From: dlg <dlg@openbsd.org>
Date: Fri, 10 Apr 2015 13:58:20 +0000 (+0000)
Subject: replace the use of ifqueues for most input queues serviced by netisr
X-Git-Url: http://artulab.com/gitweb/?a=commitdiff_plain;h=98a920fd7faa5ee5def7da7914ae31dfc8ff4d40;p=openbsd

replace the use of ifqueues for most input queues serviced by netisr
with niqueues.

this change is so big because there's a lot of code that takes
pointers to different input queues (eg, ether_input picks between
ipv4, ipv6, pppoe, arp, and mpls input queues) and falls through
to code to enqueue packets against the pointer. if i changed only
one of the input queues id have to add sepearate code paths, one
for ifqueues and one for niqueues in each of these places

by flipping all these input queues at once i can keep the currently
common code common.

testing by mpi@ sthen@ and rafael zalamena
ok mpi@ sthen@ claudio@ henning@
---

diff --git a/sys/net/if.c b/sys/net/if.c
index a7c317cde30..18bdf90b486 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.328 2015/04/10 08:48:24 mpi Exp $	*/
+/*	$OpenBSD: if.c,v 1.329 2015/04/10 13:58:20 dlg Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -126,7 +126,8 @@ void	if_attachsetup(struct ifnet *);
 void	if_attachdomain1(struct ifnet *);
 void	if_attach_common(struct ifnet *);
 
-void	if_detach_queues(struct ifnet *, struct ifqueue *);
+int	if_detach_filter(void *, const struct mbuf *);
+void	if_detach_queues(struct ifnet *, struct niqueue *);
 void	if_detached_start(struct ifnet *);
 int	if_detached_ioctl(struct ifnet *, u_long, caddr_t);
 
@@ -605,7 +606,7 @@ if_detach(struct ifnet *ifp)
 	 */
 #define IF_DETACH_QUEUES(x) \
 do { \
-	extern struct ifqueue x; \
+	extern struct niqueue x; \
 	if_detach_queues(ifp, & x); \
 } while (0)
 	IF_DETACH_QUEUES(arpintrq);
@@ -657,38 +658,31 @@ do { \
 	splx(s);
 }
 
-void
-if_detach_queues(struct ifnet *ifp, struct ifqueue *q)
+int
+if_detach_filter(void *ctx, const struct mbuf *m)
 {
-	struct mbuf *m, *prev = NULL, *next;
-	int prio;
+	struct ifnet *ifp = ctx;
 
-	for (prio = 0; prio <= IFQ_MAXPRIO; prio++) {
-		for (m = q->ifq_q[prio].head; m; m = next) {
-			next = m->m_nextpkt;
 #ifdef DIAGNOSTIC
-			if ((m->m_flags & M_PKTHDR) == 0) {
-				prev = m;
-				continue;
-			}
+	if ((m->m_flags & M_PKTHDR) == 0)
+		return (0);
 #endif
-			if (m->m_pkthdr.rcvif != ifp) {
-				prev = m;
-				continue;
-			}
 
-			if (prev)
-				prev->m_nextpkt = m->m_nextpkt;
-			else
-				q->ifq_q[prio].head = m->m_nextpkt;
-			if (q->ifq_q[prio].tail == m)
-				q->ifq_q[prio].tail = prev;
-			q->ifq_len--;
-
-			m->m_nextpkt = NULL;
-			m_freem(m);
-			IF_DROP(q);
-		}
+	return (m->m_pkthdr.rcvif == ifp);
+}
+
+void
+if_detach_queues(struct ifnet *ifp, struct niqueue *niq)
+{
+	struct mbuf *m0, *m;
+
+	m0 = niq_filter(niq, if_detach_filter, ifp);
+	while (m0 != NULL) {
+		m = m0;
+		m0 = m->m_nextpkt;
+
+		m->m_nextpkt = NULL;
+		m_freem(m);
 	}
 }
 
diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c
index 2bd2e75acae..f5e3a63a393 100644
--- a/sys/net/if_ethersubr.c
+++ b/sys/net/if_ethersubr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ethersubr.c,v 1.192 2015/04/10 08:48:24 mpi Exp $	*/
+/*	$OpenBSD: if_ethersubr.c,v 1.193 2015/04/10 13:58:20 dlg Exp $	*/
 /*	$NetBSD: if_ethersubr.c,v 1.19 1996/05/07 02:40:30 thorpej Exp $	*/
 
 /*
@@ -458,9 +458,9 @@ ether_input(struct mbuf *m, void *hdr)
 {
 	struct ifnet *ifp0, *ifp;
 	struct ether_header *eh = hdr;
-	struct ifqueue *inq;
+	struct niqueue *inq;
 	u_int16_t etype;
-	int s, llcfound = 0;
+	int llcfound = 0;
 	struct llc *l;
 	struct arpcom *ac;
 #if NTRUNK > 0
@@ -607,22 +607,15 @@ ether_input(struct mbuf *m, void *hdr)
 		}
 	}
 
-	/*
-	 * Schedule softnet interrupt and enqueue packet within the same spl.
-	 */
-	s = splnet();
 decapsulate:
-
 	switch (etype) {
 	case ETHERTYPE_IP:
-		schednetisr(NETISR_IP);
 		inq = &ipintrq;
 		break;
 
 	case ETHERTYPE_ARP:
 		if (ifp->if_flags & IFF_NOARP)
 			goto dropanyway;
-		schednetisr(NETISR_ARP);
 		inq = &arpintrq;
 		break;
 
@@ -630,14 +623,13 @@ decapsulate:
 		if (ifp->if_flags & IFF_NOARP)
 			goto dropanyway;
 		revarpinput(m);	/* XXX queue? */
-		goto done;
+		return (1);
 
 #ifdef INET6
 	/*
 	 * Schedule IPv6 software interrupt for incoming IPv6 packet.
 	 */
 	case ETHERTYPE_IPV6:
-		schednetisr(NETISR_IPV6);
 		inq = &ip6intrq;
 		break;
 #endif /* INET6 */
@@ -645,14 +637,12 @@ decapsulate:
 	case ETHERTYPE_PPPOEDISC:
 	case ETHERTYPE_PPPOE:
 #ifndef PPPOE_SERVER
-		if (m->m_flags & (M_MCAST | M_BCAST)) {
-			m_freem(m);
-			goto done;
-		}
+		if (m->m_flags & (M_MCAST | M_BCAST))
+			goto dropanyway;
 #endif
 		M_PREPEND(m, sizeof(*eh), M_DONTWAIT);
 		if (m == NULL)
-			goto done;
+			return (1);
 
 		eh_tmp = mtod(m, struct ether_header *);
 		/*
@@ -667,7 +657,7 @@ decapsulate:
 
 			if ((session = pipex_pppoe_lookup_session(m)) != NULL) {
 				pipex_pppoe_input(m, session);
-				goto done;
+				return (1);
 			}
 		}
 #endif
@@ -675,15 +665,12 @@ decapsulate:
 			inq = &pppoediscinq;
 		else
 			inq = &pppoeinq;
-
-		schednetisr(NETISR_PPPOE);
 		break;
 #endif
 #ifdef MPLS
 	case ETHERTYPE_MPLS:
 	case ETHERTYPE_MPLS_MCAST:
 		inq = &mplsintrq;
-		schednetisr(NETISR_MPLS);
 		break;
 #endif
 	default:
@@ -702,21 +689,19 @@ decapsulate:
 				m_adj(m, 6);
 				M_PREPEND(m, sizeof(*eh), M_DONTWAIT);
 				if (m == NULL)
-					goto done;
+					return (1);
 				*mtod(m, struct ether_header *) = *eh;
 				goto decapsulate;
 			}
-			goto dropanyway;
-		dropanyway:
 		default:
-			m_freem(m);
-			goto done;
+			goto dropanyway;
 		}
 	}
 
-	IF_INPUT_ENQUEUE(inq, m);
-done:
-	splx(s);
+	niq_enqueue(inq, m);
+	return (1);
+dropanyway:
+	m_freem(m);
 	return (1);
 }
 
diff --git a/sys/net/if_loop.c b/sys/net/if_loop.c
index 4e7a96dc01e..356d353df2e 100644
--- a/sys/net/if_loop.c
+++ b/sys/net/if_loop.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_loop.c,v 1.64 2015/03/14 03:38:51 jsg Exp $	*/
+/*	$OpenBSD: if_loop.c,v 1.65 2015/04/10 13:58:20 dlg Exp $	*/
 /*	$NetBSD: if_loop.c,v 1.15 1996/05/07 02:40:33 thorpej Exp $	*/
 
 /*
@@ -203,8 +203,7 @@ int
 looutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
     struct rtentry *rt)
 {
-	int s, isr;
-	struct ifqueue *ifq = 0;
+	struct niqueue *ifq = NULL;
 
 	if ((m->m_flags & M_PKTHDR) == 0)
 		panic("looutput: no header mbuf");
@@ -231,18 +230,15 @@ looutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
 
 	case AF_INET:
 		ifq = &ipintrq;
-		isr = NETISR_IP;
 		break;
 #ifdef INET6
 	case AF_INET6:
 		ifq = &ip6intrq;
-		isr = NETISR_IPV6;
 		break;
 #endif /* INET6 */
 #ifdef MPLS
 	case AF_MPLS:
 		ifq = &mplsintrq;
-		isr = NETISR_MPLS;
 		break;
 #endif /* MPLS */
 	default:
@@ -251,18 +247,13 @@ looutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
 		m_freem(m);
 		return (EAFNOSUPPORT);
 	}
-	s = splnet();
-	if (IF_QFULL(ifq)) {
-		IF_DROP(ifq);
-		m_freem(m);
-		splx(s);
+
+	if (niq_enqueue(ifq, m) != 0)
 		return (ENOBUFS);
-	}
-	IF_ENQUEUE(ifq, m);
-	schednetisr(isr);
+
 	ifp->if_ipackets++;
 	ifp->if_ibytes += m->m_pkthdr.len;
-	splx(s);
+
 	return (0);
 }
 
diff --git a/sys/net/if_mpe.c b/sys/net/if_mpe.c
index 02714c7693f..a868a27cf28 100644
--- a/sys/net/if_mpe.c
+++ b/sys/net/if_mpe.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: if_mpe.c,v 1.42 2015/03/26 11:02:44 mpi Exp $ */
+/* $OpenBSD: if_mpe.c,v 1.43 2015/04/10 13:58:20 dlg Exp $ */
 
 /*
  * Copyright (c) 2008 Pierre-Yves Ritschard <pyr@spootnik.org>
@@ -371,7 +371,7 @@ mpe_input(struct mbuf *m, struct ifnet *ifp, struct sockaddr_mpls *smpls,
     u_int8_t ttl)
 {
 	struct ip	*ip;
-	int		 s, hlen;
+	int		 hlen;
 
 	/* label -> AF lookup */
 
@@ -408,10 +408,8 @@ mpe_input(struct mbuf *m, struct ifnet *ifp, struct sockaddr_mpls *smpls,
 	if (ifp && ifp->if_bpf)
 		bpf_mtap_af(ifp->if_bpf, AF_INET, m, BPF_DIRECTION_IN);
 #endif
-	s = splnet();
-	IF_INPUT_ENQUEUE(&ipintrq, m);
-	schednetisr(NETISR_IP);
-	splx(s);
+
+	niq_enqueue(&ipintrq, m);
 }
 
 #ifdef INET6
@@ -420,7 +418,6 @@ mpe_input6(struct mbuf *m, struct ifnet *ifp, struct sockaddr_mpls *smpls,
     u_int8_t ttl)
 {
 	struct ip6_hdr *ip6hdr;
-	int s;
 
 	/* label -> AF lookup */
 
@@ -443,9 +440,7 @@ mpe_input6(struct mbuf *m, struct ifnet *ifp, struct sockaddr_mpls *smpls,
 	if (ifp && ifp->if_bpf)
 		bpf_mtap_af(ifp->if_bpf, AF_INET6, m, BPF_DIRECTION_IN);
 #endif
-	s = splnet();
-	IF_INPUT_ENQUEUE(&ip6intrq, m);
-	schednetisr(NETISR_IPV6);
-	splx(s);
+
+	niq_enqueue(&ip6intrq, m);
 }
 #endif	/* INET6 */
diff --git a/sys/net/if_ppp.c b/sys/net/if_ppp.c
index 0e5e67c278f..e924037f9c9 100644
--- a/sys/net/if_ppp.c
+++ b/sys/net/if_ppp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ppp.c,v 1.81 2015/03/18 12:23:15 dlg Exp $	*/
+/*	$OpenBSD: if_ppp.c,v 1.82 2015/04/10 13:58:20 dlg Exp $	*/
 /*	$NetBSD: if_ppp.c,v 1.39 1997/05/17 21:11:59 christos Exp $	*/
 
 /*
@@ -232,7 +232,7 @@ ppp_clone_create(struct if_clone *ifc, int unit)
     sc->sc_if.if_output = pppoutput;
     sc->sc_if.if_start = ppp_ifstart;
     IFQ_SET_MAXLEN(&sc->sc_if.if_snd, IFQ_MAXLEN);
-    IFQ_SET_MAXLEN(&sc->sc_inq, IFQ_MAXLEN);
+    mq_init(&sc->sc_inq, IFQ_MAXLEN, IPL_NET);
     IFQ_SET_MAXLEN(&sc->sc_fastq, IFQ_MAXLEN);
     IFQ_SET_MAXLEN(&sc->sc_rawq, IFQ_MAXLEN);
     IFQ_SET_READY(&sc->sc_if.if_snd);
@@ -329,12 +329,8 @@ pppdealloc(struct ppp_softc *sc)
 	    break;
 	m_freem(m);
     }
-    for (;;) {
-	IF_DEQUEUE(&sc->sc_inq, m);
-	if (m == NULL)
-	    break;
+    while ((m = mq_dequeue(&sc->sc_inq)) != NULL)
 	m_freem(m);
-    }
     for (;;) {
 	IF_DEQUEUE(&sc->sc_fastq, m);
 	if (m == NULL)
@@ -398,7 +394,7 @@ pppioctl(struct ppp_softc *sc, u_long cmd, caddr_t data, int flag,
 
     switch (cmd) {
     case FIONREAD:
-	*(int *)data = IFQ_LEN(&sc->sc_inq);
+	*(int *)data = mq_len(&sc->sc_inq);
 	break;
 
     case PPPIOCGUNIT:
@@ -1225,7 +1221,6 @@ static void
 ppp_inproc(struct ppp_softc *sc, struct mbuf *m)
 {
     struct ifnet *ifp = &sc->sc_if;
-    struct ifqueue *inq;
     int s, ilen, xlen, proto, rv;
     u_char *cp, adrs, ctrl;
     struct mbuf *mp, *dmp = NULL;
@@ -1462,44 +1457,44 @@ ppp_inproc(struct ppp_softc *sc, struct mbuf *m)
 	m->m_pkthdr.len -= PPP_HDRLEN;
 	m->m_data += PPP_HDRLEN;
 	m->m_len -= PPP_HDRLEN;
-	schednetisr(NETISR_IP);
-	inq = &ipintrq;
+
+	if (niq_enqueue(&ipintrq, m) != 0)
+		rv = 0; /* failure */
+	else
+		rv = 1; /* ipintrq success */
 	break;
 
     default:
 	/*
 	 * Some other protocol - place on input queue for read().
 	 */
-	inq = &sc->sc_inq;
-	rv = 1;
+	if (mq_enqueue(&sc->sc_inq, m) != 0) {
+		if_congestion();
+		rv = 0; /* failure */
+	} else
+		rv = 2; /* input queue */
 	break;
     }
 
-    /*
-     * Put the packet on the appropriate input queue.
-     */
-    s = splnet();
-    if (IF_QFULL(inq)) {
-	IF_DROP(inq);
-	splx(s);
+    if (rv == 0) {
+	/* failure */
 	if (sc->sc_flags & SC_DEBUG)
 	    printf("%s: input queue full\n", ifp->if_xname);
 	ifp->if_iqdrops++;
-	if_congestion();
-	goto bad;
+	goto dropped;
     }
-    IF_ENQUEUE(inq, m);
-    splx(s);
+
     ifp->if_ipackets++;
     ifp->if_ibytes += ilen;
 
-    if (rv)
+    if (rv == 2)
 	(*sc->sc_ctlp)(sc);
 
     return;
 
  bad:
     m_freem(m);
+ dropped:
     sc->sc_if.if_ierrors++;
     sc->sc_stats.ppp_ierrors++;
 }
diff --git a/sys/net/if_pppoe.c b/sys/net/if_pppoe.c
index dc26afcd6c2..ebfe565b31c 100644
--- a/sys/net/if_pppoe.c
+++ b/sys/net/if_pppoe.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: if_pppoe.c,v 1.44 2015/03/14 03:38:51 jsg Exp $ */
+/* $OpenBSD: if_pppoe.c,v 1.45 2015/04/10 13:58:20 dlg Exp $ */
 /* $NetBSD: if_pppoe.c,v 1.51 2003/11/28 08:56:48 keihan Exp $ */
 
 /*
@@ -48,6 +48,7 @@
 #include <net/if_types.h>
 #include <net/if_sppp.h>
 #include <net/if_pppoe.h>
+#include <net/netisr.h>
 #include <netinet/in.h>
 #include <netinet/if_ether.h>
 
@@ -146,8 +147,8 @@ struct pppoe_softc {
 };
 
 /* incoming traffic will be queued here */
-struct ifqueue pppoediscinq;
-struct ifqueue pppoeinq;
+struct niqueue pppoediscinq = NIQUEUE_INITIALIZER(IFQ_MAXLEN, NETISR_PPPOE);
+struct niqueue pppoeinq = NIQUEUE_INITIALIZER(IFQ_MAXLEN, NETISR_PPPOE);
 
 /* input routines */
 static void pppoe_disc_input(struct mbuf *);
@@ -200,9 +201,6 @@ pppoeattach(int count)
 {
 	LIST_INIT(&pppoe_softc_list);
 	if_clone_attach(&pppoe_cloner);
-
-	IFQ_SET_MAXLEN(&pppoediscinq, IFQ_MAXLEN);
-	IFQ_SET_MAXLEN(&pppoeinq, IFQ_MAXLEN);
 }
 
 /* Create a new interface. */
@@ -359,27 +357,14 @@ void
 pppoeintr(void)
 {
 	struct mbuf *m;
-	int s;
 
 	splsoftassert(IPL_SOFTNET);
-	
-	for (;;) {
-		s = splnet();
-		IF_DEQUEUE(&pppoediscinq, m);
-		splx(s);
-		if (m == NULL)
-			break;
+
+	while ((m = niq_dequeue(&pppoediscinq)) != NULL)
 		pppoe_disc_input(m);
-	}
 
-	for (;;) {
-		s = splnet();
-		IF_DEQUEUE(&pppoeinq, m);
-		splx(s);
-		if (m == NULL)
-			break;
+	while ((m = niq_dequeue(&pppoeinq)) != NULL)
 		pppoe_data_input(m);
-	}
 }
 
 /* Analyze and handle a single received packet while not in session state. */
diff --git a/sys/net/if_pppoe.h b/sys/net/if_pppoe.h
index 8251cf0f369..4aa6ad2aae7 100644
--- a/sys/net/if_pppoe.h
+++ b/sys/net/if_pppoe.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_pppoe.h,v 1.5 2008/08/28 13:10:54 brad Exp $ */
+/*	$OpenBSD: if_pppoe.h,v 1.6 2015/04/10 13:58:20 dlg Exp $ */
 /*	$NetBSD: if_pppoe.h,v 1.5 2003/11/28 08:56:48 keihan Exp $ */
 
 /*
@@ -66,8 +66,8 @@ struct pppoeconnectionstate {
 
 #ifdef _KERNEL
 
-extern struct ifqueue pppoediscinq;
-extern struct ifqueue pppoeinq;
+extern struct niqueue pppoediscinq;
+extern struct niqueue pppoeinq;
 
 void pppoeintr(void);
 
diff --git a/sys/net/if_pppvar.h b/sys/net/if_pppvar.h
index e25986d498f..130fc7a07d3 100644
--- a/sys/net/if_pppvar.h
+++ b/sys/net/if_pppvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_pppvar.h,v 1.15 2003/12/07 15:41:27 markus Exp $	*/
+/*	$OpenBSD: if_pppvar.h,v 1.16 2015/04/10 13:58:20 dlg Exp $	*/
 /*	$NetBSD: if_pppvar.h,v 1.5 1997/01/03 07:23:29 mikel Exp $	*/
 /*
  * if_pppvar.h - private structures and declarations for PPP.
@@ -98,7 +98,7 @@ struct ppp_softc {
 	u_int16_t sc_mru;		/* max receive unit */
 	pid_t	sc_xfer;		/* used in transferring unit */
 	struct	ifqueue sc_rawq;	/* received packets */
-	struct	ifqueue sc_inq;		/* queue of input packets for daemon */
+	struct	mbuf_queue sc_inq;	/* queue of input packets for daemon */
 	struct	ifqueue sc_fastq;	/* interactive output packet q */
 	struct	mbuf *sc_togo;		/* output packet ready to go */
 	struct	mbuf *sc_npqueue;	/* output packets not to be sent yet */
diff --git a/sys/net/if_pppx.c b/sys/net/if_pppx.c
index c2ed2f72206..00de01c7498 100644
--- a/sys/net/if_pppx.c
+++ b/sys/net/if_pppx.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_pppx.c,v 1.36 2015/02/10 21:56:10 miod Exp $ */
+/*	$OpenBSD: if_pppx.c,v 1.37 2015/04/10 13:58:20 dlg Exp $ */
 
 /*
  * Copyright (c) 2010 Claudio Jeker <claudio@openbsd.org>
@@ -317,9 +317,9 @@ pppxwrite(dev_t dev, struct uio *uio, int ioflag)
 /*	struct pppx_dev *pxd = pppx_dev2pxd(dev);	*/
 	struct pppx_hdr *th;
 	struct mbuf *top, **mp, *m;
-	struct ifqueue *ifq;
+	struct niqueue *ifq;
 	int tlen, mlen;
-	int isr, s, error = 0;
+	int error = 0;
 
 	if (uio->uio_resid < sizeof(*th) || uio->uio_resid > MCLBYTES)
 		return (EMSGSIZE);
@@ -381,12 +381,10 @@ pppxwrite(dev_t dev, struct uio *uio, int ioflag)
 	switch (ntohl(th->pppx_proto)) {
 	case AF_INET:
 		ifq = &ipintrq;
-		isr = NETISR_IP;
 		break;
 #ifdef INET6
 	case AF_INET6:
 		ifq = &ip6intrq;
-		isr = NETISR_IPV6;
 		break;
 #endif
 	default:
@@ -394,16 +392,8 @@ pppxwrite(dev_t dev, struct uio *uio, int ioflag)
 		return (EAFNOSUPPORT);
 	}
 
-	s = splnet();
-	if (IF_QFULL(ifq)) {
-		IF_DROP(ifq);
-		splx(s);
-		m_freem(top);
+	if (niq_enqueue(ifq, m) != 0)
 		return (ENOBUFS);
-	}
-	IF_ENQUEUE(ifq, top);
-	schednetisr(isr);
-	splx(s);
 
 	return (error);
 }
diff --git a/sys/net/if_spppsubr.c b/sys/net/if_spppsubr.c
index 30bc0aa994f..0c6b9fb4a0d 100644
--- a/sys/net/if_spppsubr.c
+++ b/sys/net/if_spppsubr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_spppsubr.c,v 1.131 2015/03/18 12:23:15 dlg Exp $	*/
+/*	$OpenBSD: if_spppsubr.c,v 1.132 2015/04/10 13:58:20 dlg Exp $	*/
 /*
  * Synchronous PPP/Cisco link level subroutines.
  * Keepalive protocol implemented in both Cisco and PPP modes.
@@ -437,11 +437,10 @@ void
 sppp_input(struct ifnet *ifp, struct mbuf *m)
 {
 	struct ppp_header ht;
-	struct ifqueue *inq = 0;
+	struct niqueue *inq = NULL;
 	struct sppp *sp = (struct sppp *)ifp;
 	struct timeval tv;
 	int debug = ifp->if_flags & IFF_DEBUG;
-	int s;
 
 	if (ifp->if_flags & IFF_UP) {
 		/* Count received bytes, add hardware framing */
@@ -458,9 +457,10 @@ sppp_input(struct ifnet *ifp, struct mbuf *m)
 			    SPP_FMT "input packet is too small, %d bytes\n",
 			    SPP_ARGS(ifp), m->m_pkthdr.len);
 	  drop:
+		m_freem (m);
+	  dropped:
 		++ifp->if_ierrors;
 		++ifp->if_iqdrops;
-		m_freem (m);
 		return;
 	}
 
@@ -538,7 +538,6 @@ sppp_input(struct ifnet *ifp, struct mbuf *m)
 			return;
 		case PPP_IP:
 			if (sp->state[IDX_IPCP] == STATE_OPENED) {
-				schednetisr (NETISR_IP);
 				inq = &ipintrq;
 				sp->pp_last_activity = tv.tv_sec;
 			}
@@ -551,7 +550,6 @@ sppp_input(struct ifnet *ifp, struct mbuf *m)
 			return;
 		case PPP_IPV6:
 			if (sp->state[IDX_IPV6CP] == STATE_OPENED) {
-				schednetisr (NETISR_IPV6);
 				inq = &ip6intrq;
 				sp->pp_last_activity = tv.tv_sec;
 			}
@@ -580,12 +578,10 @@ sppp_input(struct ifnet *ifp, struct mbuf *m)
 			m_freem (m);
 			return;
 		case ETHERTYPE_IP:
-			schednetisr (NETISR_IP);
 			inq = &ipintrq;
 			break;
 #ifdef INET6
 		case ETHERTYPE_IPV6:
-			schednetisr (NETISR_IPV6);
 			inq = &ip6intrq;
 			break;
 #endif
@@ -605,20 +601,13 @@ sppp_input(struct ifnet *ifp, struct mbuf *m)
 	if (! (ifp->if_flags & IFF_UP) || ! inq)
 		goto drop;
 
-	/* Check queue. */
-	s = splnet();
-	if (IF_QFULL (inq)) {
+	if (niq_enqueue(inq, m) != 0) {
 		/* Queue overflow. */
-		IF_DROP(inq);
-		splx(s);
 		if (debug)
 			log(LOG_DEBUG, SPP_FMT "protocol queue overflow\n",
 				SPP_ARGS(ifp));
-		if_congestion();
-		goto drop;
+		goto dropped;
 	}
-	IF_ENQUEUE(inq, m);
-	splx(s);
 }
 
 /*
diff --git a/sys/net/if_tun.c b/sys/net/if_tun.c
index 6bea0923eac..542ba2d3883 100644
--- a/sys/net/if_tun.c
+++ b/sys/net/if_tun.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_tun.c,v 1.135 2015/04/01 14:29:54 mpi Exp $	*/
+/*	$OpenBSD: if_tun.c,v 1.136 2015/04/10 13:58:20 dlg Exp $	*/
 /*	$NetBSD: if_tun.c,v 1.24 1996/05/07 02:40:48 thorpej Exp $	*/
 
 /*
@@ -780,10 +780,9 @@ tunwrite(dev_t dev, struct uio *uio, int ioflag)
 {
 	struct tun_softc	*tp;
 	struct ifnet		*ifp;
-	struct ifqueue		*ifq;
+	struct niqueue		*ifq;
 	u_int32_t		*th;
 	struct mbuf		*top, **mp, *m;
-	int			 isr;
 	int			 error=0, s, tlen, mlen;
 
 	if ((tp = tun_lookup(minor(dev))) == NULL)
@@ -887,12 +886,10 @@ tunwrite(dev_t dev, struct uio *uio, int ioflag)
 	switch (ntohl(*th)) {
 	case AF_INET:
 		ifq = &ipintrq;
-		isr = NETISR_IP;
 		break;
 #ifdef INET6
 	case AF_INET6:
 		ifq = &ip6intrq;
-		isr = NETISR_IPV6;
 		break;
 #endif
 	default:
@@ -900,20 +897,14 @@ tunwrite(dev_t dev, struct uio *uio, int ioflag)
 		return (EAFNOSUPPORT);
 	}
 
-	s = splnet();
-	if (IF_QFULL(ifq)) {
-		IF_DROP(ifq);
-		splx(s);
+	if (niq_enqueue(ifq, m) != 0) {
 		ifp->if_collisions++;
-		m_freem(top);
-		if_congestion();
 		return (ENOBUFS);
 	}
-	IF_ENQUEUE(ifq, top);
-	schednetisr(isr);
+
 	ifp->if_ipackets++;
 	ifp->if_ibytes += top->m_pkthdr.len;
-	splx(s);
+
 	return (error);
 }
 
diff --git a/sys/net/pipex.c b/sys/net/pipex.c
index 0caa9a10d05..51fc8c3b2f2 100644
--- a/sys/net/pipex.c
+++ b/sys/net/pipex.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pipex.c,v 1.67 2015/04/10 11:02:12 dlg Exp $	*/
+/*	$OpenBSD: pipex.c,v 1.68 2015/04/10 13:58:20 dlg Exp $	*/
 
 /*-
  * Copyright (c) 2009 Internet Initiative Japan Inc.
@@ -1127,7 +1127,7 @@ pipex_ip_input(struct mbuf *m0, struct pipex_session *session)
 {
 	struct ifnet *ifp;
 	struct ip *ip;
-	int s, len;
+	int len;
 	int is_idle;
 
 	/* change recvif */
@@ -1189,28 +1189,21 @@ pipex_ip_input(struct mbuf *m0, struct pipex_session *session)
 		bpf_mtap_af(ifp->if_bpf, AF_INET, m0, BPF_DIRECTION_IN);
 #endif
 
-	s = splnet();
-	if (IF_QFULL(&ipintrq)) {
-		IF_DROP(&ipintrq);
+	if (niq_enqueue(&ipintrq, m0) != 0) {
 		ifp->if_collisions++;
-		if_congestion();
-		splx(s);
-		goto drop;
+		goto dropped;
 	}
-	IF_ENQUEUE(&ipintrq, m0);
-	schednetisr(NETISR_IP);
 
 	ifp->if_ipackets++;
 	ifp->if_ibytes += len;
 	session->stat.ipackets++;
 	session->stat.ibytes += len;
 
-	splx(s);
-
 	return;
 drop:
 	if (m0 != NULL)
 		m_freem(m0);
+dropped:
 	session->stat.ierrors++;
 }
 
@@ -1220,7 +1213,7 @@ pipex_ip6_input(struct mbuf *m0, struct pipex_session *session)
 {
 	struct ifnet *ifp;
 	struct ip6_hdr *ip6;
-	int s, len;
+	int len;
 
 	/* change recvif */
 	m0->m_pkthdr.rcvif = session->pipex_iface->ifnet_this;
@@ -1263,28 +1256,18 @@ pipex_ip6_input(struct mbuf *m0, struct pipex_session *session)
 		bpf_mtap_af(ifp->if_bpf, AF_INET6, m0, BPF_DIRECTION_IN);
 #endif
 
-	s = splnet();
-	if (IF_QFULL(&ip6intrq)) {
-		IF_DROP(&ip6intrq);
+	if (niq_enqueue(&ip6intrq, m0) != 0) {
 		ifp->if_collisions++;
-		if_congestion();
-		splx(s);
-		goto drop;
+		goto dropped;
 	}
-	IF_ENQUEUE(&ip6intrq, m0);
-	schednetisr(NETISR_IPV6);
 
 	ifp->if_ipackets++;
 	ifp->if_ibytes += len;
 	session->stat.ipackets++;
 	session->stat.ibytes += len;
 
-	splx(s);
-
 	return;
-drop:
-	if (m0 != NULL)
-		m_freem(m0);
+dropped:
 	session->stat.ierrors++;
 }
 #endif
diff --git a/sys/net/ppp_tty.c b/sys/net/ppp_tty.c
index 6b104298023..e2cc9fd5178 100644
--- a/sys/net/ppp_tty.c
+++ b/sys/net/ppp_tty.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ppp_tty.c,v 1.31 2015/02/10 21:56:10 miod Exp $	*/
+/*	$OpenBSD: ppp_tty.c,v 1.32 2015/04/10 13:58:20 dlg Exp $	*/
 /*	$NetBSD: ppp_tty.c,v 1.12 1997/03/24 21:23:10 christos Exp $	*/
 
 /*
@@ -303,7 +303,9 @@ pppread(struct tty *tp, struct uio *uio, int flag)
 	    splx(s);
 	    return 0;
 	}
-	if (!IF_IS_EMPTY(&sc->sc_inq))
+	/* Get the packet from the input queue */
+	m0 = mq_dequeue(&sc->sc_inq);
+	if (m0 != NULL)
 	    break;
 	if ((tp->t_state & TS_CARR_ON) == 0 && (tp->t_cflag & CLOCAL) == 0
 	    && (tp->t_state & TS_ISOPEN)) {
@@ -323,9 +325,6 @@ pppread(struct tty *tp, struct uio *uio, int flag)
 
     /* Pull place-holder byte out of canonical queue */
     getc(&tp->t_canq);
-
-    /* Get the packet from the input queue */
-    IF_DEQUEUE(&sc->sc_inq, m0);
     splx(s);
 
     for (m = m0; m && uio->uio_resid; m = m->m_next)
diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c
index 1a66993b4dd..43a88e8843e 100644
--- a/sys/netinet/if_ether.c
+++ b/sys/netinet/if_ether.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ether.c,v 1.149 2015/03/24 12:58:43 mpi Exp $	*/
+/*	$OpenBSD: if_ether.c,v 1.150 2015/04/10 13:58:20 dlg Exp $	*/
 /*	$NetBSD: if_ether.c,v 1.31 1996/05/11 12:59:58 mycroft Exp $	*/
 
 /*
@@ -57,6 +57,7 @@
 #include <net/if_dl.h>
 #include <net/route.h>
 #include <net/if_types.h>
+#include <net/netisr.h>
 
 #include <netinet/in.h>
 #include <netinet/in_var.h>
@@ -97,7 +98,8 @@ void in_arpinput(struct mbuf *);
 
 LIST_HEAD(, llinfo_arp) llinfo_arp;
 struct	pool arp_pool;		/* pool for llinfo_arp structures */
-struct	ifqueue arpintrq;
+/* XXX hate magic numbers */
+struct	niqueue arpintrq = NIQUEUE_INITIALIZER(50, NETISR_ARP);
 int	arp_inuse, arp_allocated;
 int	arp_maxtries = 5;
 int	arpinit_done;
@@ -159,7 +161,6 @@ arp_rtrequest(int req, struct rtentry *rt)
 		arpinit_done = 1;
 		pool_init(&arp_pool, sizeof(struct llinfo_arp), 0, 0, 0, "arp",
 		    NULL);
-		IFQ_SET_MAXLEN(&arpintrq, 50);	/* XXX hate magic numbers */
 		/*
 		 * We generate expiration times from time.tv_sec
 		 * so avoid accidently creating permanent routes.
@@ -497,14 +498,9 @@ arpintr(void)
 {
 	struct mbuf *m;
 	struct arphdr *ar;
-	int s, len;
+	int len;
 
-	for (;;) {
-		s = splnet();
-		IF_DEQUEUE(&arpintrq, m);
-		splx(s);
-		if (m == NULL)
-			break;
+	while ((m = niq_dequeue(&arpintrq)) != NULL) {
 #ifdef DIAGNOSTIC
 		if ((m->m_flags & M_PKTHDR) == 0)
 			panic("arpintr");
diff --git a/sys/netinet/if_ether.h b/sys/netinet/if_ether.h
index 96760712c62..aa7394da822 100644
--- a/sys/netinet/if_ether.h
+++ b/sys/netinet/if_ether.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ether.h,v 1.55 2015/03/24 12:58:43 mpi Exp $	*/
+/*	$OpenBSD: if_ether.h,v 1.56 2015/04/10 13:58:20 dlg Exp $	*/
 /*	$NetBSD: if_ether.h,v 1.22 1996/05/11 13:00:00 mycroft Exp $	*/
 
 /*
@@ -188,7 +188,7 @@ struct sockaddr_inarp {
 extern u_int8_t etherbroadcastaddr[ETHER_ADDR_LEN];
 extern u_int8_t ether_ipmulticast_min[ETHER_ADDR_LEN];
 extern u_int8_t ether_ipmulticast_max[ETHER_ADDR_LEN];
-extern struct ifqueue arpintrq;
+extern struct niqueue arpintrq;
 
 void	arpwhohas(struct arpcom *, struct in_addr *);
 void	arpintr(void);
diff --git a/sys/netinet/in.h b/sys/netinet/in.h
index 1319d89fa49..b7b55eb154d 100644
--- a/sys/netinet/in.h
+++ b/sys/netinet/in.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: in.h,v 1.112 2015/02/09 12:18:19 claudio Exp $	*/
+/*	$OpenBSD: in.h,v 1.113 2015/04/10 13:58:20 dlg Exp $	*/
 /*	$NetBSD: in.h,v 1.20 1996/02/13 23:41:47 christos Exp $	*/
 
 /*
@@ -785,7 +785,7 @@ __END_DECLS
 
 #ifdef _KERNEL
 extern	   int inetctlerrmap[];
-extern	   struct ifqueue ipintrq;	/* ip packet input queue */
+extern	   struct niqueue ipintrq;	/* ip packet input queue */
 extern	   struct in_addr zeroin_addr;
 
 struct mbuf;
diff --git a/sys/netinet/ip_divert.c b/sys/netinet/ip_divert.c
index a1f85cfe60e..1b36f12dcd6 100644
--- a/sys/netinet/ip_divert.c
+++ b/sys/netinet/ip_divert.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ip_divert.c,v 1.32 2015/01/24 00:29:06 deraadt Exp $ */
+/*      $OpenBSD: ip_divert.c,v 1.33 2015/04/10 13:58:20 dlg Exp $ */
 
 /*
  * Copyright (c) 2009 Michele Marchetto <michele@openbsd.org>
@@ -82,11 +82,10 @@ int
 divert_output(struct inpcb *inp, struct mbuf *m, struct mbuf *nam,
     struct mbuf *control)
 {
-	struct ifqueue *inq;
 	struct sockaddr_in *sin;
 	struct socket *so;
 	struct ifaddr *ifa;
-	int s, error = 0, min_hdrlen = 0, dir;
+	int error = 0, min_hdrlen = 0, dir;
 	struct ip *ip;
 	u_int16_t off;
 
@@ -149,8 +148,6 @@ divert_output(struct inpcb *inp, struct mbuf *m, struct mbuf *nam,
 		}
 		m->m_pkthdr.rcvif = ifa->ifa_ifp;
 
-		inq = &ipintrq;
-
 		/*
 		 * Recalculate IP and protocol checksums for the inbound packet
 		 * since the userspace application may have modified the packet
@@ -160,10 +157,7 @@ divert_output(struct inpcb *inp, struct mbuf *m, struct mbuf *nam,
 		ip->ip_sum = in_cksum(m, off);
 		in_proto_cksum_out(m, NULL);
 
-		s = splnet();
-		IF_INPUT_ENQUEUE(inq, m);
-		schednetisr(NETISR_IP);
-		splx(s);
+		niq_enqueue(&ipintrq, m);
 	} else {
 		error = ip_output(m, NULL, &inp->inp_route,
 		    IP_ALLOWBROADCAST | IP_RAWOUTPUT, NULL, NULL, 0);
diff --git a/sys/netinet/ip_ether.c b/sys/netinet/ip_ether.c
index 9c7def0c6a9..1633b5cd696 100644
--- a/sys/netinet/ip_ether.c
+++ b/sys/netinet/ip_ether.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_ether.c,v 1.70 2014/12/19 17:14:40 tedu Exp $  */
+/*	$OpenBSD: ip_ether.c,v 1.71 2015/04/10 13:58:20 dlg Exp $  */
 /*
  * The author of this code is Angelos D. Keromytis (kermit@adk.gr)
  *
@@ -280,8 +280,6 @@ void
 mplsip_decap(struct mbuf *m, int iphlen)
 {
 	struct gif_softc *sc;
-	struct ifqueue *ifq;
-	int s;
 
 	etheripstat.etherip_ipackets++;
 
@@ -330,22 +328,12 @@ mplsip_decap(struct mbuf *m, int iphlen)
 	pf_pkt_addr_changed(m);
 #endif
 
-	ifq = &mplsintrq;
-	s = splnet();
-	if (IF_QFULL(ifq)) {
-		IF_DROP(ifq);
-		m_freem(m);
+	if (niq_enqueue(&mplsintrq, m) != 0) {
 		etheripstat.etherip_qfull++;
-		splx(s);
 
 		DPRINTF(("mplsip_input(): packet dropped because of full "
 		    "queue\n"));
-		return;
 	}
-	IF_ENQUEUE(ifq, m);
-	schednetisr(NETISR_MPLS);
-	splx(s);
-	return;
 }
 #endif
 
diff --git a/sys/netinet/ip_gre.c b/sys/netinet/ip_gre.c
index 10d4b6c90a0..c2038ccaa26 100644
--- a/sys/netinet/ip_gre.c
+++ b/sys/netinet/ip_gre.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ip_gre.c,v 1.53 2015/03/18 01:12:16 mcbride Exp $ */
+/*      $OpenBSD: ip_gre.c,v 1.54 2015/04/10 13:58:20 dlg Exp $ */
 /*	$NetBSD: ip_gre.c,v 1.9 1999/10/25 19:18:11 drochner Exp $ */
 
 /*
@@ -93,8 +93,7 @@ int
 gre_input2(struct mbuf *m, int hlen, u_char proto)
 {
 	struct greip *gip;
-	int s;
-	struct ifqueue *ifq;
+	struct niqueue *ifq;
 	struct gre_softc *sc;
 	u_short flags;
 	u_int af;
@@ -168,7 +167,6 @@ gre_input2(struct mbuf *m, int hlen, u_char proto)
 #ifdef INET6
 		case ETHERTYPE_IPV6:
 		        ifq = &ip6intrq;
-			schednetisr(NETISR_IPV6);
 			af = AF_INET6;
 			break;
 #endif
@@ -181,7 +179,6 @@ gre_input2(struct mbuf *m, int hlen, u_char proto)
 		case ETHERTYPE_MPLS:
 		case ETHERTYPE_MPLS_MCAST:
 			ifq = &mplsintrq;
-			schednetisr(NETISR_MPLS);
 			af = AF_MPLS;
 			break;
 #endif
@@ -209,9 +206,7 @@ gre_input2(struct mbuf *m, int hlen, u_char proto)
 	pf_pkt_addr_changed(m);
 #endif
 
-	s = splnet();		/* possible */
-	IF_INPUT_ENQUEUE(ifq, m);
-	splx(s);
+	niq_enqueue(ifq, m);
 
 	return (1);	/* packet is done, no further processing needed */
 }
@@ -271,9 +266,8 @@ gre_mobile_input(struct mbuf *m, ...)
 {
 	struct ip *ip;
 	struct mobip_h *mip;
-	struct ifqueue *ifq;
 	struct gre_softc *sc;
-	int hlen, s;
+	int hlen;
 	va_list ap;
 	u_char osrc = 0;
 	int msiz;
@@ -339,16 +333,12 @@ gre_mobile_input(struct mbuf *m, ...)
 	ip->ip_sum = 0;
 	ip->ip_sum = in_cksum(m,(ip->ip_hl << 2));
 
-	ifq = &ipintrq;
-
 #if NBPFILTER > 0
         if (sc->sc_if.if_bpf)
 		bpf_mtap_af(sc->sc_if.if_bpf, AF_INET, m, BPF_DIRECTION_IN);
 #endif
 
-	s = splnet();       /* possible */
-	IF_INPUT_ENQUEUE(ifq, m);
-	splx(s);
+	niq_enqueue(&ipintrq, m);
 }
 
 /*
diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c
index d0525fb3e66..38b140395ab 100644
--- a/sys/netinet/ip_input.c
+++ b/sys/netinet/ip_input.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_input.c,v 1.247 2015/03/14 03:38:52 jsg Exp $	*/
+/*	$OpenBSD: ip_input.c,v 1.248 2015/04/10 13:58:20 dlg Exp $	*/
 /*	$NetBSD: ip_input.c,v 1.30 1996/03/16 23:53:58 christos Exp $	*/
 
 /*
@@ -49,6 +49,7 @@
 #include <net/if_var.h>
 #include <net/if_dl.h>
 #include <net/route.h>
+#include <net/netisr.h>
 
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
@@ -113,7 +114,7 @@ int	ip_frags = 0;
 
 int *ipctl_vars[IPCTL_MAXID] = IPCTL_VARS;
 
-struct	ifqueue ipintrq;
+struct niqueue ipintrq = NIQUEUE_INITIALIZER(IFQ_MAXLEN, NETISR_IP);
 
 struct pool ipqent_pool;
 struct pool ipq_pool;
@@ -169,7 +170,6 @@ ip_init(void)
 		    pr->pr_protocol < IPPROTO_MAX)
 			ip_protox[pr->pr_protocol] = pr - inetsw;
 	LIST_INIT(&ipq);
-	IFQ_SET_MAXLEN(&ipintrq, IFQ_MAXLEN);
 	if (ip_mtudisc != 0)
 		ip_mtudisc_timeout_q =
 		    rt_timer_queue_create(ip_mtudisc_timeout);
@@ -192,18 +192,12 @@ void
 ipintr(void)
 {
 	struct mbuf *m;
-	int s;
 
-	for (;;) {
-		/*
-		 * Get next datagram off input queue and get IP header
-		 * in first mbuf.
-		 */
-		s = splnet();
-		IF_DEQUEUE(&ipintrq, m);
-		splx(s);
-		if (m == NULL)
-			return;
+	/*
+	 * Get next datagram off input queue and get IP header
+	 * in first mbuf.
+	 */
+	while ((m = niq_dequeue(&ipintrq)) != NULL) {
 #ifdef	DIAGNOSTIC
 		if ((m->m_flags & M_PKTHDR) == 0)
 			panic("ipintr no HDR");
@@ -1616,7 +1610,7 @@ ip_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, void *newp,
 				       ipsec_def_comp,
 				       sizeof(ipsec_def_comp)));
 	case IPCTL_IFQUEUE:
-	        return (sysctl_ifq(name + 1, namelen - 1,
+	        return (sysctl_niq(name + 1, namelen - 1,
 		    oldp, oldlenp, newp, newlen, &ipintrq));
 	case IPCTL_STATS:
 		return (sysctl_rdstruct(oldp, oldlenp, newp,
diff --git a/sys/netinet/ip_ipip.c b/sys/netinet/ip_ipip.c
index d427512beb5..4128e3d3990 100644
--- a/sys/netinet/ip_ipip.c
+++ b/sys/netinet/ip_ipip.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_ipip.c,v 1.56 2014/12/19 17:14:40 tedu Exp $ */
+/*	$OpenBSD: ip_ipip.c,v 1.57 2015/04/10 13:58:20 dlg Exp $ */
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
@@ -146,15 +146,14 @@ ipip_input(struct mbuf *m, int iphlen, struct ifnet *gifp, int proto)
 	struct sockaddr_in *sin;
 	struct ifnet *ifp;
 	struct ifaddr *ifa;
-	struct ifqueue *ifq = NULL;
+	struct niqueue *ifq = NULL;
 	struct ip *ipo;
 	u_int rdomain;
 #ifdef INET6
 	struct sockaddr_in6 *sin6;
 	struct ip6_hdr *ip6;
 #endif
-	int isr;
-	int mode, hlen, s;
+	int mode, hlen;
 	u_int8_t itos, otos;
 	u_int8_t v;
 	sa_family_t af;
@@ -352,13 +351,11 @@ ipip_input(struct mbuf *m, int iphlen, struct ifnet *gifp, int proto)
 	switch (proto) {
 	case IPPROTO_IPV4:
 		ifq = &ipintrq;
-		isr = NETISR_IP;
 		af = AF_INET;
 		break;
 #ifdef INET6
 	case IPPROTO_IPV6:
 		ifq = &ip6intrq;
-		isr = NETISR_IPV6;
 		af = AF_INET6;
 		break;
 #endif
@@ -374,23 +371,12 @@ ipip_input(struct mbuf *m, int iphlen, struct ifnet *gifp, int proto)
 	pf_pkt_addr_changed(m);
 #endif
 
-	s = splnet();			/* isn't it already? */
-	if (IF_QFULL(ifq)) {
-		IF_DROP(ifq);
-		m_freem(m);
+	if (niq_enqueue(ifq, m) != 0) {
 		ipipstat.ipips_qfull++;
-
-		splx(s);
-
 		DPRINTF(("ipip_input(): packet dropped because of full "
 		    "queue\n"));
 		return;
 	}
-
-	IF_ENQUEUE(ifq, m);
-	schednetisr(isr);
-	splx(s);
-	return;
 }
 
 int
diff --git a/sys/netinet/ipsec_input.c b/sys/netinet/ipsec_input.c
index 9973fcdc155..498c940bd33 100644
--- a/sys/netinet/ipsec_input.c
+++ b/sys/netinet/ipsec_input.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ipsec_input.c,v 1.127 2015/03/26 12:21:37 mikeb Exp $	*/
+/*	$OpenBSD: ipsec_input.c,v 1.128 2015/04/10 13:58:20 dlg Exp $	*/
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
@@ -708,28 +708,18 @@ ah4_input(struct mbuf *m, ...)
 int
 ah4_input_cb(struct mbuf *m, ...)
 {
-	struct ifqueue *ifq = &ipintrq;
-	int s = splnet();
-
 	/*
 	 * Interface pointer is already in first mbuf; chop off the
 	 * `outer' header and reschedule.
 	 */
 
-	if (IF_QFULL(ifq)) {
-		IF_DROP(ifq);
+	if (niq_enqueue(&ipintrq, m) != 0) {
 		ahstat.ahs_qfull++;
-		splx(s);
-
-		m_freem(m);
 		DPRINTF(("ah4_input_cb(): dropped packet because of full "
 		    "IP queue\n"));
 		return ENOBUFS;
 	}
 
-	IF_ENQUEUE(ifq, m);
-	schednetisr(NETISR_IP);
-	splx(s);
 	return 0;
 }
 
@@ -764,27 +754,17 @@ esp4_input(struct mbuf *m, ...)
 int
 esp4_input_cb(struct mbuf *m, ...)
 {
-	struct ifqueue *ifq = &ipintrq;
-	int s = splnet();
-
 	/*
 	 * Interface pointer is already in first mbuf; chop off the
 	 * `outer' header and reschedule.
 	 */
-	if (IF_QFULL(ifq)) {
-		IF_DROP(ifq);
+	if (niq_enqueue(&ipintrq, m) != 0) {
 		espstat.esps_qfull++;
-		splx(s);
-
-		m_freem(m);
 		DPRINTF(("esp4_input_cb(): dropped packet because of full "
 		    "IP queue\n"));
 		return ENOBUFS;
 	}
 
-	IF_ENQUEUE(ifq, m);
-	schednetisr(NETISR_IP);
-	splx(s);
 	return 0;
 }
 
@@ -806,27 +786,16 @@ ipcomp4_input(struct mbuf *m, ...)
 int
 ipcomp4_input_cb(struct mbuf *m, ...)
 {
-	struct ifqueue *ifq = &ipintrq;
-	int s = splnet();
-
 	/*
 	 * Interface pointer is already in first mbuf; chop off the
 	 * `outer' header and reschedule.
 	 */
-	if (IF_QFULL(ifq)) {
-		IF_DROP(ifq);
+	if (niq_enqueue(&ipintrq, m) != 0) {
 		ipcompstat.ipcomps_qfull++;
-		splx(s);
-
-		m_freem(m);
 		DPRINTF(("ipcomp4_input_cb(): dropped packet because of full IP queue\n"));
 		return ENOBUFS;
 	}
 
-	IF_ENQUEUE(ifq, m);
-	schednetisr(NETISR_IP);
-	splx(s);
-
 	return 0;
 }
 
diff --git a/sys/netinet6/in6.h b/sys/netinet6/in6.h
index 66559187492..33e2b633ed4 100644
--- a/sys/netinet6/in6.h
+++ b/sys/netinet6/in6.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: in6.h,v 1.80 2015/02/09 12:23:22 claudio Exp $	*/
+/*	$OpenBSD: in6.h,v 1.81 2015/04/10 13:58:20 dlg Exp $	*/
 /*	$KAME: in6.h,v 1.83 2001/03/29 02:55:07 jinmei Exp $	*/
 
 /*
@@ -418,7 +418,7 @@ typedef	__socklen_t	socklen_t;	/* length type for network syscalls */
 
 #ifdef _KERNEL
 extern	u_char inet6ctlerrmap[];
-extern	struct ifqueue ip6intrq;	/* IP6 packet input queue */
+extern	struct niqueue ip6intrq;	/* IP6 packet input queue */
 extern	struct in6_addr zeroin6_addr;
 
 struct mbuf;
diff --git a/sys/netinet6/ip6_divert.c b/sys/netinet6/ip6_divert.c
index b82e6c4e362..af13323d65e 100644
--- a/sys/netinet6/ip6_divert.c
+++ b/sys/netinet6/ip6_divert.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ip6_divert.c,v 1.32 2015/01/24 00:29:06 deraadt Exp $ */
+/*      $OpenBSD: ip6_divert.c,v 1.33 2015/04/10 13:58:20 dlg Exp $ */
 
 /*
  * Copyright (c) 2009 Michele Marchetto <michele@openbsd.org>
@@ -86,11 +86,10 @@ int
 divert6_output(struct inpcb *inp, struct mbuf *m, struct mbuf *nam,
     struct mbuf *control)
 {
-	struct ifqueue *inq;
 	struct sockaddr_in6 *sin6;
 	struct socket *so;
 	struct ifaddr *ifa;
-	int s, error = 0, min_hdrlen = 0, nxt = 0, off, dir;
+	int error = 0, min_hdrlen = 0, nxt = 0, off, dir;
 	struct ip6_hdr *ip6;
 
 	m->m_pkthdr.rcvif = NULL;
@@ -159,8 +158,6 @@ divert6_output(struct inpcb *inp, struct mbuf *m, struct mbuf *nam,
 		}
 		m->m_pkthdr.rcvif = ifa->ifa_ifp;
 
-		inq = &ip6intrq;
-
 		/*
 		 * Recalculate the protocol checksum for the inbound packet
 		 * since the userspace application may have modified the packet
@@ -168,10 +165,7 @@ divert6_output(struct inpcb *inp, struct mbuf *m, struct mbuf *nam,
 		 */
 		in6_proto_cksum_out(m, NULL);
 
-		s = splnet();
-		IF_INPUT_ENQUEUE(inq, m);
-		schednetisr(NETISR_IPV6);
-		splx(s);
+		niq_enqueue(&ip6intrq, m); /* return error on q full? */
 	} else {
 		error = ip6_output(m, NULL, &inp->inp_route6,
 		    IP_ALLOWBROADCAST | IP_RAWOUTPUT, NULL, NULL, NULL);
diff --git a/sys/netinet6/ip6_input.c b/sys/netinet6/ip6_input.c
index f4c9351a704..9643cdb61dd 100644
--- a/sys/netinet6/ip6_input.c
+++ b/sys/netinet6/ip6_input.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip6_input.c,v 1.140 2015/03/14 03:38:52 jsg Exp $	*/
+/*	$OpenBSD: ip6_input.c,v 1.141 2015/04/10 13:58:20 dlg Exp $	*/
 /*	$KAME: ip6_input.c,v 1.188 2001/03/29 05:34:31 itojun Exp $	*/
 
 /*
@@ -113,7 +113,7 @@
 #endif
 
 struct in6_ifaddrhead in6_ifaddr;
-struct ifqueue ip6intrq;
+struct niqueue ip6intrq = NIQUEUE_INITIALIZER(IFQ_MAXLEN, NETISR_IPV6);
 
 struct ip6stat ip6stat;
 
@@ -144,7 +144,6 @@ ip6_init(void)
 		    pr->pr_protocol && pr->pr_protocol != IPPROTO_RAW &&
 		    pr->pr_protocol < IPPROTO_MAX)
 			ip6_protox[pr->pr_protocol] = pr - inet6sw;
-	IFQ_SET_MAXLEN(&ip6intrq, IFQ_MAXLEN);
 	TAILQ_INIT(&in6_ifaddr);
 	ip6_randomid_init();
 	nd6_init();
@@ -168,17 +167,10 @@ ip6_init2(void *dummy)
 void
 ip6intr(void)
 {
-	int s;
 	struct mbuf *m;
 
-	for (;;) {
-		s = splnet();
-		IF_DEQUEUE(&ip6intrq, m);
-		splx(s);
-		if (m == NULL)
-			return;
+	while ((m = niq_dequeue(&ip6intrq)) != NULL)
 		ip6_input(m);
-	}
 }
 
 extern struct	route_in6 ip6_forward_rt;
@@ -1452,7 +1444,7 @@ ip6_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp,
 		}
 		return (error);
 	case IPV6CTL_IFQUEUE:
-		return (sysctl_ifq(name + 1, namelen - 1,
+		return (sysctl_niq(name + 1, namelen - 1,
 		    oldp, oldlenp, newp, newlen, &ip6intrq));
 	default:
 		if (name[0] < IPV6CTL_MAXID)
diff --git a/sys/netmpls/mpls.h b/sys/netmpls/mpls.h
index 638dd82ab0a..fa06237bd7b 100644
--- a/sys/netmpls/mpls.h
+++ b/sys/netmpls/mpls.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mpls.h,v 1.30 2015/03/26 11:02:44 mpi Exp $	*/
+/*	$OpenBSD: mpls.h,v 1.31 2015/04/10 13:58:20 dlg Exp $	*/
 
 /*
  * Copyright (C) 1999, 2000 and 2001 AYAME Project, WIDE Project.
@@ -160,7 +160,7 @@ void	mpe_input6(struct mbuf *, struct ifnet *, struct sockaddr_mpls *,
 extern int mpls_raw_usrreq(struct socket *, int, struct mbuf *,
 			struct mbuf *, struct mbuf *, struct proc *);
 
-extern struct ifqueue	mplsintrq;	/* MPLS input queue */
+extern struct niqueue	mplsintrq;	/* MPLS input queue */
 extern int		mpls_defttl;
 extern int		mpls_mapttl_ip;
 extern int		mpls_mapttl_ip6;
diff --git a/sys/netmpls/mpls_input.c b/sys/netmpls/mpls_input.c
index 5328aecf122..1929786cbdc 100644
--- a/sys/netmpls/mpls_input.c
+++ b/sys/netmpls/mpls_input.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mpls_input.c,v 1.42 2014/12/23 03:24:08 tedu Exp $	*/
+/*	$OpenBSD: mpls_input.c,v 1.43 2015/04/10 13:58:20 dlg Exp $	*/
 
 /*
  * Copyright (c) 2008 Claudio Jeker <claudio@openbsd.org>
@@ -40,7 +40,7 @@
 
 #include <netmpls/mpls.h>
 
-struct ifqueue	mplsintrq;
+struct niqueue	mplsintrq = NIQUEUE_INITIALIZER(IFQ_MAXLEN, NETISR_MPLS);
 
 #ifdef MPLS_DEBUG
 #define MPLS_LABEL_GET(l)	((ntohl((l) & MPLS_LABEL_MASK)) >> MPLS_LABEL_OFFSET)
@@ -57,22 +57,15 @@ struct mbuf	*mpls_do_error(struct mbuf *, int, int, int);
 void
 mpls_init(void)
 {
-	IFQ_SET_MAXLEN(&mplsintrq, IFQ_MAXLEN);
 }
 
 void
 mplsintr(void)
 {
 	struct mbuf *m;
-	int s;
 
-	for (;;) {
-		/* Get next datagram of input queue */
-		s = splnet();
-		IF_DEQUEUE(&mplsintrq, m);
-		splx(s);
-		if (m == NULL)
-			return;
+	/* Get next datagram of input queue */
+	while ((m = niq_dequeue(&mplsintrq)) != NULL) {
 #ifdef DIAGNOSTIC
 		if ((m->m_flags & M_PKTHDR) == 0)
 			panic("mplsintr no HDR");
@@ -91,7 +84,7 @@ mpls_input(struct mbuf *m)
 	struct rtentry *rt = NULL;
 	struct rt_mpls *rt_mpls;
 	u_int8_t ttl;
-	int i, s, hasbos;
+	int i, hasbos;
 
 	if (!ISSET(ifp->if_xflags, IFXF_MPLS)) {
 		m_freem(m);
@@ -158,10 +151,7 @@ mpls_input(struct mbuf *m)
 do_v4:
 					if (mpls_ip_adjttl(m, ttl))
 						goto done;
-					s = splnet();
-					IF_INPUT_ENQUEUE(&ipintrq, m);
-					schednetisr(NETISR_IP);
-					splx(s);
+					niq_enqueue(&ipintrq, m);
 					goto done;
 				}
 				continue;
@@ -171,10 +161,7 @@ do_v4:
 do_v6:
 					if (mpls_ip6_adjttl(m, ttl))
 						goto done;
-					s = splnet();
-					IF_INPUT_ENQUEUE(&ip6intrq, m);
-					schednetisr(NETISR_IPV6);
-					splx(s);
+					niq_enqueue(&ip6intrq, m);
 					goto done;
 				}
 				continue;
@@ -241,19 +228,13 @@ do_v6:
 			case AF_INET:
 				if (mpls_ip_adjttl(m, ttl))
 					break;
-				s = splnet();
-				IF_INPUT_ENQUEUE(&ipintrq, m);
-				schednetisr(NETISR_IP);
-				splx(s);
+				niq_enqueue(&ipintrq, m);
 				break;
 #ifdef INET6
 			case AF_INET6:
 				if (mpls_ip6_adjttl(m, ttl))
 					break;
-				s = splnet();
-				IF_INPUT_ENQUEUE(&ip6intrq, m);
-				schednetisr(NETISR_IPV6);
-				splx(s);
+				niq_enqueue(&ip6intrq, m);
 				break;
 #endif
 			default:
diff --git a/sys/netmpls/mpls_raw.c b/sys/netmpls/mpls_raw.c
index bb95029f569..a3165155f4c 100644
--- a/sys/netmpls/mpls_raw.c
+++ b/sys/netmpls/mpls_raw.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mpls_raw.c,v 1.10 2014/12/05 15:50:04 mpi Exp $	*/
+/*	$OpenBSD: mpls_raw.c,v 1.11 2015/04/10 13:58:20 dlg Exp $	*/
 
 /*
  * Copyright (C) 1999, 2000 and 2001 AYAME Project, WIDE Project.
@@ -135,7 +135,7 @@ mpls_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, void *newp,
 
 	switch (name[0]) {
 	case MPLSCTL_IFQUEUE:
-		return (sysctl_ifq(name + 1, namelen - 1,
+		return (sysctl_niq(name + 1, namelen - 1,
 		    oldp, oldlenp, newp, newlen, &mplsintrq));
 	default:
 		return sysctl_int_arr(mplsctl_vars, name, namelen,