From 0cd5c6fa3ba6ed18b7d438feb06ecd9305b858db Mon Sep 17 00:00:00 2001
From: mvs <mvs@openbsd.org>
Date: Tue, 28 Jun 2022 08:01:40 +0000
Subject: [PATCH] Introduce `pipexoutq' mbuf(9) queue, and put outgoing
 pipex(4) related PPPOE packets within. Do (*if_output)() calls within netisr
 handler with netlock held.

We can't predict netlock state when pipex(4) related (*if_qstart)()
handlers called. This means we can't use netlock within pppac_qstart()
and pppx_if_qstart() handlers.

ok bluhm@
---
 sys/net/if.c     |  6 ++++-
 sys/net/netisr.h |  4 +++-
 sys/net/pipex.c  | 62 +++++++++++++++++++++++++++++++++++++++---------
 3 files changed, 59 insertions(+), 13 deletions(-)

diff --git a/sys/net/if.c b/sys/net/if.c
index c12943d6af9..8a29aec0f03 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.654 2022/06/27 15:11:23 jan Exp $	*/
+/*	$OpenBSD: if.c,v 1.655 2022/06/28 08:01:40 mvs Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -913,6 +913,10 @@ if_netisr(void *unused)
 #if NBRIDGE > 0
 		if (n & (1 << NETISR_BRIDGE))
 			bridgeintr();
+#endif
+#ifdef PIPEX
+		if (n & (1 << NETISR_PIPEX))
+			pipexintr();
 #endif
 		t |= n;
 	}
diff --git a/sys/net/netisr.h b/sys/net/netisr.h
index 6e90c277d2f..616296357a2 100644
--- a/sys/net/netisr.h
+++ b/sys/net/netisr.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: netisr.h,v 1.56 2022/04/28 16:56:39 bluhm Exp $	*/
+/*	$OpenBSD: netisr.h,v 1.57 2022/06/28 08:01:40 mvs Exp $	*/
 /*	$NetBSD: netisr.h,v 1.12 1995/08/12 23:59:24 mycroft Exp $	*/
 
 /*
@@ -45,6 +45,7 @@
 #define	NETISR_PFSYNC	5		/* for pfsync "immediate" tx */
 #define	NETISR_ARP	18		/* same as AF_LINK */
 #define	NETISR_IPV6	24		/* same as AF_INET6 */
+#define	NETISR_PIPEX	27		/* for pipex processing */
 #define	NETISR_PPP	28		/* for PPP processing */
 #define	NETISR_BRIDGE	29		/* for bridge processing */
 #define	NETISR_SWITCH	31		/* for switch dataplane */
@@ -65,6 +66,7 @@ void	pppintr(void);
 void	bridgeintr(void);
 void	switchintr(void);
 void	pfsyncintr(void);
+void	pipexintr(void);
 
 #define	schednetisr(anisr)						\
 do {									\
diff --git a/sys/net/pipex.c b/sys/net/pipex.c
index 393a7d563d7..7eef55ff153 100644
--- a/sys/net/pipex.c
+++ b/sys/net/pipex.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pipex.c,v 1.141 2022/06/26 22:51:58 mvs Exp $ */
+/*	$OpenBSD: pipex.c,v 1.142 2022/06/28 08:01:40 mvs Exp $ */
 
 /*-
  * Copyright (c) 2009 Internet Initiative Japan Inc.
@@ -52,6 +52,7 @@
 #include <net/route.h>
 #include <net/ppp_defs.h>
 #include <net/ppp-comp.h>
+#include <net/netisr.h>
 
 #include "pf.h"
 #if NPF > 0
@@ -106,6 +107,9 @@ struct radix_node_head	*pipex_rd_head6 = NULL;	/* [L] */
 struct timeout pipex_timer_ch;		/* callout timer context */
 int pipex_prune = 1;			/* [I] walk list every seconds */
 
+struct mbuf_queue pipexoutq = MBUF_QUEUE_INITIALIZER(
+    IFQ_MAXLEN, IPL_SOFTNET);
+
 /* borrow an mbuf pkthdr field */
 #define ph_ppp_proto ether_vtag
 
@@ -193,6 +197,46 @@ pipex_ioctl(void *ownersc, u_long cmd, caddr_t data)
 	return (ret);
 }
 
+/************************************************************************
+ * Software Interrupt Handler
+ ************************************************************************/
+
+void
+pipexintr(void)
+{
+	struct mbuf_list ml;
+	struct mbuf *m;
+	struct pipex_session *session;
+
+	NET_ASSERT_LOCKED();
+
+	mq_delist(&pipexoutq, &ml);
+
+	while ((m = ml_dequeue(&ml)) != NULL) {
+		struct ifnet *ifp;
+
+		session = m->m_pkthdr.ph_cookie;
+
+		ifp = if_get(session->proto.pppoe.over_ifidx);
+		if (ifp != NULL) {
+			struct pipex_pppoe_header *pppoe;
+			int len;
+
+			pppoe = mtod(m, struct pipex_pppoe_header *);
+			len = ntohs(pppoe->length);
+			ifp->if_output(ifp, m, &session->peer.sa, NULL);
+			counters_pkt(session->stat_counters, pxc_opackets,
+			    pxc_obytes, len);
+		} else {
+			m_freem(m);
+			counters_inc(session->stat_counters, pxc_oerrors);
+		}
+		if_put(ifp);
+
+		pipex_rele_session(session);
+	}
+}
+
 /************************************************************************
  * Session management functions
  ************************************************************************/
@@ -1259,7 +1303,6 @@ Static void
 pipex_pppoe_output(struct mbuf *m0, struct pipex_session *session)
 {
 	struct pipex_pppoe_header *pppoe;
-	struct ifnet *ifp;
 	int len, padlen;
 
 	/* save length for pppoe header */
@@ -1286,18 +1329,15 @@ pipex_pppoe_output(struct mbuf *m0, struct pipex_session *session)
 	pppoe->length = htons(len);
 
 	m0->m_pkthdr.ph_ifidx = session->proto.pppoe.over_ifidx;
+	refcnt_take(&session->pxs_refcnt);
+	m0->m_pkthdr.ph_cookie = session;
 	m0->m_flags &= ~(M_BCAST|M_MCAST);
 
-	ifp = if_get(session->proto.pppoe.over_ifidx);
-	if (ifp != NULL) {
-		ifp->if_output(ifp, m0, &session->peer.sa, NULL);
-		counters_pkt(session->stat_counters, pxc_opackets,
-		    pxc_obytes, len);
-	} else {
-		m_freem(m0);
+	if (mq_enqueue(&pipexoutq, m0) != 0) {
 		counters_inc(session->stat_counters, pxc_oerrors);
-	}
-	if_put(ifp);
+		pipex_rele_session(session);
+	} else
+		schednetisr(NETISR_PIPEX);
 }
 #endif /* PIPEX_PPPOE */
 
-- 
2.20.1