From 7019ae976ad91f2bc281f67a336deb30aa8168c1 Mon Sep 17 00:00:00 2001 From: bluhm Date: Sun, 14 Apr 2024 20:46:27 +0000 Subject: [PATCH] Run raw IP input in parallel. Running raw IPv4 input with shared net lock in parallel is less complex than UDP. Especially there is no socket splicing. New ip_deliver() may run with shared or exclusive net lock. The last parameter indicates the mode. If is is running with shared netlock and encounters a protocol that needs exclusive lock, the packet is queued. Old ip_ours() always queued the packet. Now it calls ip_deliver() with shared net lock, and if that cannot handle the packet completely, the packet is queued and later processed with exclusive net lock. In case of an IPv6 header chain, that switches from shared to exclusive processing, the next protocol and mbuf offset are stored in a mbuf tag. OK mvs@ --- sys/net/if_bridge.c | 4 +- sys/netinet/in_proto.c | 6 +-- sys/netinet/ip_input.c | 102 ++++++++++++++++++++++++++++++--------- sys/netinet/ip_var.h | 9 +++- sys/netinet6/ip6_input.c | 21 ++++---- sys/sys/mbuf.h | 5 +- sys/sys/protosw.h | 3 +- 7 files changed, 105 insertions(+), 45 deletions(-) diff --git a/sys/net/if_bridge.c b/sys/net/if_bridge.c index 51c65e8fa48..154f7de7841 100644 --- a/sys/net/if_bridge.c +++ b/sys/net/if_bridge.c @@ -1,4 +1,4 @@ -/* $OpenBSD: if_bridge.c,v 1.369 2024/02/13 12:22:09 bluhm Exp $ */ +/* $OpenBSD: if_bridge.c,v 1.370 2024/04/14 20:46:27 bluhm Exp $ */ /* * Copyright (c) 1999, 2000 Jason L. Wright (jason@thought.net) @@ -1600,7 +1600,7 @@ bridge_ipsec(struct ifnet *ifp, struct ether_header *eh, int hassnap, off); tdb_unref(tdb); if (prot != IPPROTO_DONE) - ip_deliver(&m, &hlen, prot, af); + ip_deliver(&m, &hlen, prot, af, 0); return (1); } else { tdb_unref(tdb); diff --git a/sys/netinet/in_proto.c b/sys/netinet/in_proto.c index 9748baff7e0..a7ba54e52cf 100644 --- a/sys/netinet/in_proto.c +++ b/sys/netinet/in_proto.c @@ -1,4 +1,4 @@ -/* $OpenBSD: in_proto.c,v 1.103 2024/01/11 14:15:12 bluhm Exp $ */ +/* $OpenBSD: in_proto.c,v 1.104 2024/04/14 20:46:27 bluhm Exp $ */ /* $NetBSD: in_proto.c,v 1.14 1996/02/18 18:58:32 christos Exp $ */ /* @@ -210,7 +210,7 @@ const struct protosw inetsw[] = { .pr_type = SOCK_RAW, .pr_domain = &inetdomain, .pr_protocol = IPPROTO_RAW, - .pr_flags = PR_ATOMIC|PR_ADDR, + .pr_flags = PR_ATOMIC|PR_ADDR|PR_MPINPUT, .pr_input = rip_input, .pr_ctloutput = rip_ctloutput, .pr_usrreqs = &rip_usrreqs, @@ -377,7 +377,7 @@ const struct protosw inetsw[] = { /* raw wildcard */ .pr_type = SOCK_RAW, .pr_domain = &inetdomain, - .pr_flags = PR_ATOMIC|PR_ADDR, + .pr_flags = PR_ATOMIC|PR_ADDR|PR_MPINPUT, .pr_input = rip_input, .pr_ctloutput = rip_ctloutput, .pr_usrreqs = &rip_usrreqs, diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c index 9c273238f39..25f9cc5ebd1 100644 --- a/sys/netinet/ip_input.c +++ b/sys/netinet/ip_input.c @@ -1,4 +1,4 @@ -/* $OpenBSD: ip_input.c,v 1.391 2024/02/28 10:57:20 bluhm Exp $ */ +/* $OpenBSD: ip_input.c,v 1.392 2024/04/14 20:46:27 bluhm Exp $ */ /* $NetBSD: ip_input.c,v 1.30 1996/03/16 23:53:58 christos Exp $ */ /* @@ -245,6 +245,30 @@ ip_ours(struct mbuf **mp, int *offp, int nxt, int af) if (af != AF_UNSPEC) return nxt; + nxt = ip_deliver(mp, offp, nxt, AF_INET, 1); + if (nxt == IPPROTO_DONE) + return IPPROTO_DONE; + + /* save values for later, use after dequeue */ + if (*offp != sizeof(struct ip)) { + struct m_tag *mtag; + struct ipoffnxt *ion; + + /* mbuf tags are expensive, but only used for header options */ + mtag = m_tag_get(PACKET_TAG_IP_OFFNXT, sizeof(*ion), + M_NOWAIT); + if (mtag == NULL) { + ipstat_inc(ips_idropped); + m_freemp(mp); + return IPPROTO_DONE; + } + ion = (struct ipoffnxt *)(mtag + 1); + ion->ion_off = *offp; + ion->ion_nxt = nxt; + + m_tag_prepend(*mp, mtag); + } + niq_enqueue(&ipintrq, *mp); *mp = NULL; return IPPROTO_DONE; @@ -260,18 +284,31 @@ ipintr(void) struct mbuf *m; while ((m = niq_dequeue(&ipintrq)) != NULL) { - struct ip *ip; + struct m_tag *mtag; int off, nxt; #ifdef DIAGNOSTIC if ((m->m_flags & M_PKTHDR) == 0) panic("ipintr no HDR"); #endif - ip = mtod(m, struct ip *); - off = ip->ip_hl << 2; - nxt = ip->ip_p; + mtag = m_tag_find(m, PACKET_TAG_IP_OFFNXT, NULL); + if (mtag != NULL) { + struct ipoffnxt *ion; + + ion = (struct ipoffnxt *)(mtag + 1); + off = ion->ion_off; + nxt = ion->ion_nxt; + + m_tag_delete(m, mtag); + } else { + struct ip *ip; - nxt = ip_deliver(&m, &off, nxt, AF_INET); + ip = mtod(m, struct ip *); + off = ip->ip_hl << 2; + nxt = ip->ip_p; + } + + nxt = ip_deliver(&m, &off, nxt, AF_INET, 0); KASSERT(nxt == IPPROTO_DONE); } } @@ -675,15 +712,11 @@ ip_fragcheck(struct mbuf **mp, int *offp) #endif int -ip_deliver(struct mbuf **mp, int *offp, int nxt, int af) +ip_deliver(struct mbuf **mp, int *offp, int nxt, int af, int shared) { - const struct protosw *psw; - int naf = af; #ifdef INET6 int nest = 0; -#endif /* INET6 */ - - NET_ASSERT_LOCKED_EXCLUSIVE(); +#endif /* * Tell launch routine the next header @@ -691,13 +724,41 @@ ip_deliver(struct mbuf **mp, int *offp, int nxt, int af) IPSTAT_INC(delivered); while (nxt != IPPROTO_DONE) { + const struct protosw *psw; + int naf; + + switch (af) { + case AF_INET: + psw = &inetsw[ip_protox[nxt]]; + break; +#ifdef INET6 + case AF_INET6: + psw = &inet6sw[ip6_protox[nxt]]; + break; +#endif + } + if (shared && !ISSET(psw->pr_flags, PR_MPINPUT)) { + /* delivery not finished, decrement counter, queue */ + switch (af) { + case AF_INET: + counters_dec(ipcounters, ips_delivered); + break; +#ifdef INET6 + case AF_INET6: + counters_dec(ip6counters, ip6s_delivered); + break; +#endif + } + break; + } + #ifdef INET6 if (af == AF_INET6 && ip6_hdrnestlimit && (++nest > ip6_hdrnestlimit)) { ip6stat_inc(ip6s_toomanyhdr); goto bad; } -#endif /* INET6 */ +#endif /* * protection against faulty packet - there should be @@ -716,7 +777,7 @@ ip_deliver(struct mbuf **mp, int *offp, int nxt, int af) } } /* Otherwise, just fall through and deliver the packet */ -#endif /* IPSEC */ +#endif switch (nxt) { case IPPROTO_IPV4: @@ -728,17 +789,10 @@ ip_deliver(struct mbuf **mp, int *offp, int nxt, int af) naf = AF_INET6; ip6stat_inc(ip6s_delivered); break; -#endif /* INET6 */ - } - switch (af) { - case AF_INET: - psw = &inetsw[ip_protox[nxt]]; - break; -#ifdef INET6 - case AF_INET6: - psw = &inet6sw[ip6_protox[nxt]]; +#endif + default: + naf = af; break; -#endif /* INET6 */ } nxt = (*psw->pr_input)(mp, offp, nxt, af); af = naf; diff --git a/sys/netinet/ip_var.h b/sys/netinet/ip_var.h index 2c0d45379df..c5b92847561 100644 --- a/sys/netinet/ip_var.h +++ b/sys/netinet/ip_var.h @@ -1,4 +1,4 @@ -/* $OpenBSD: ip_var.h,v 1.114 2024/03/05 09:45:13 bluhm Exp $ */ +/* $OpenBSD: ip_var.h,v 1.115 2024/04/14 20:46:27 bluhm Exp $ */ /* $NetBSD: ip_var.h,v 1.16 1996/02/13 23:43:20 christos Exp $ */ /* @@ -198,6 +198,11 @@ struct ipq { struct in_addr ipq_src, ipq_dst; }; +struct ipoffnxt { + int ion_off; + int ion_nxt; +}; + /* flags passed to ip_output */ #define IP_FORWARDING 0x1 /* most of ip header exists */ #define IP_RAWOUTPUT 0x2 /* raw ip header exists */ @@ -254,7 +259,7 @@ int ip_sysctl(int *, u_int, void *, size_t *, void *, size_t); void ip_savecontrol(struct inpcb *, struct mbuf **, struct ip *, struct mbuf *); int ip_input_if(struct mbuf **, int *, int, int, struct ifnet *); -int ip_deliver(struct mbuf **, int *, int, int); +int ip_deliver(struct mbuf **, int *, int, int, int); void ip_forward(struct mbuf *, struct ifnet *, struct rtentry *, int); int rip_ctloutput(int, struct socket *, int, int, struct mbuf *); void rip_init(void); diff --git a/sys/netinet6/ip6_input.c b/sys/netinet6/ip6_input.c index 112fe60376d..5f4e991e0f8 100644 --- a/sys/netinet6/ip6_input.c +++ b/sys/netinet6/ip6_input.c @@ -1,4 +1,4 @@ -/* $OpenBSD: ip6_input.c,v 1.259 2024/02/28 10:57:20 bluhm Exp $ */ +/* $OpenBSD: ip6_input.c,v 1.260 2024/04/14 20:46:27 bluhm Exp $ */ /* $KAME: ip6_input.c,v 1.188 2001/03/29 05:34:31 itojun Exp $ */ /* @@ -166,11 +166,6 @@ ip6_init(void) #endif } -struct ip6_offnxt { - int ion_off; - int ion_nxt; -}; - /* * Enqueue packet for local delivery. Queuing is used as a boundary * between the network layer (input/forward path) running with @@ -190,10 +185,14 @@ ip6_ours(struct mbuf **mp, int *offp, int nxt, int af) if (af != AF_UNSPEC) return nxt; + nxt = ip_deliver(mp, offp, nxt, AF_INET6, 1); + if (nxt == IPPROTO_DONE) + return IPPROTO_DONE; + /* save values for later, use after dequeue */ if (*offp != sizeof(struct ip6_hdr)) { struct m_tag *mtag; - struct ip6_offnxt *ion; + struct ipoffnxt *ion; /* mbuf tags are expensive, but only used for header options */ mtag = m_tag_get(PACKET_TAG_IP6_OFFNXT, sizeof(*ion), @@ -203,7 +202,7 @@ ip6_ours(struct mbuf **mp, int *offp, int nxt, int af) m_freemp(mp); return IPPROTO_DONE; } - ion = (struct ip6_offnxt *)(mtag + 1); + ion = (struct ipoffnxt *)(mtag + 1); ion->ion_off = *offp; ion->ion_nxt = nxt; @@ -234,9 +233,9 @@ ip6intr(void) #endif mtag = m_tag_find(m, PACKET_TAG_IP6_OFFNXT, NULL); if (mtag != NULL) { - struct ip6_offnxt *ion; + struct ipoffnxt *ion; - ion = (struct ip6_offnxt *)(mtag + 1); + ion = (struct ipoffnxt *)(mtag + 1); off = ion->ion_off; nxt = ion->ion_nxt; @@ -248,7 +247,7 @@ ip6intr(void) off = sizeof(struct ip6_hdr); nxt = ip6->ip6_nxt; } - nxt = ip_deliver(&m, &off, nxt, AF_INET6); + nxt = ip_deliver(&m, &off, nxt, AF_INET6, 0); KASSERT(nxt == IPPROTO_DONE); } } diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h index 866a65b1d0e..b458e0e897a 100644 --- a/sys/sys/mbuf.h +++ b/sys/sys/mbuf.h @@ -1,4 +1,4 @@ -/* $OpenBSD: mbuf.h,v 1.262 2024/02/21 13:42:06 bluhm Exp $ */ +/* $OpenBSD: mbuf.h,v 1.263 2024/04/14 20:46:27 bluhm Exp $ */ /* $NetBSD: mbuf.h,v 1.19 1996/02/09 18:25:14 christos Exp $ */ /* @@ -471,6 +471,8 @@ struct m_tag *m_tag_next(struct mbuf *, struct m_tag *); #define PACKET_TAG_IPSEC_IN_DONE 0x0001 /* IPsec applied, in */ #define PACKET_TAG_IPSEC_OUT_DONE 0x0002 /* IPsec applied, out */ #define PACKET_TAG_IPSEC_FLOWINFO 0x0004 /* IPsec flowinfo */ +#define PACKET_TAG_IP_OFFNXT 0x0010 /* IPv4 offset and next proto */ +#define PACKET_TAG_IP6_OFFNXT 0x0020 /* IPv6 offset and next proto */ #define PACKET_TAG_WIREGUARD 0x0040 /* WireGuard data */ #define PACKET_TAG_GRE 0x0080 /* GRE processing done */ #define PACKET_TAG_DLT 0x0100 /* data link layer type */ @@ -479,7 +481,6 @@ struct m_tag *m_tag_next(struct mbuf *, struct m_tag *); #define PACKET_TAG_SRCROUTE 0x1000 /* IPv4 source routing options */ #define PACKET_TAG_TUNNEL 0x2000 /* Tunnel endpoint address */ #define PACKET_TAG_CARP_BAL_IP 0x4000 /* carp(4) ip balanced marker */ -#define PACKET_TAG_IP6_OFFNXT 0x8000 /* IPv6 offset and next proto */ #define MTAG_BITS \ ("\20\1IPSEC_IN_DONE\2IPSEC_OUT_DONE\3IPSEC_FLOWINFO" \ diff --git a/sys/sys/protosw.h b/sys/sys/protosw.h index c6af33751fc..319cc5c0909 100644 --- a/sys/sys/protosw.h +++ b/sys/sys/protosw.h @@ -1,4 +1,4 @@ -/* $OpenBSD: protosw.h,v 1.65 2024/02/03 22:50:09 mvs Exp $ */ +/* $OpenBSD: protosw.h,v 1.66 2024/04/14 20:46:27 bluhm Exp $ */ /* $NetBSD: protosw.h,v 1.10 1996/04/09 20:55:32 cgd Exp $ */ /*- @@ -132,6 +132,7 @@ struct protosw { #define PR_ABRTACPTDIS 0x0020 /* abort on accept(2) to disconnected socket */ #define PR_SPLICE 0x0040 /* socket splicing is possible */ +#define PR_MPINPUT 0x0080 /* input runs with shared netlock */ /* * The arguments to usrreq are: -- 2.20.1