From 6c89734d1e67558b2a8a278059fa173c7c95a125 Mon Sep 17 00:00:00 2001
From: patrick <patrick@openbsd.org>
Date: Fri, 7 Jul 2023 10:23:39 +0000
Subject: [PATCH] The per-VQ MSI-X interrupt handler needs to sync DMA mappings
 in the same way that the shared interrupt handler does.  This is one of the
 requirements of virtio_dequeue(), as specified in its comment above.

Without the DMA sync, it will not see a new entry on the ring and
return.  Since the interrupt is edge-triggered there won't be another
one and we'll get stuck.

ok dv@
---
 sys/dev/pci/virtio_pci.c | 14 ++++++++++----
 sys/dev/pv/virtio.c      | 33 +++++++++++++++++++--------------
 sys/dev/pv/virtiovar.h   |  3 ++-
 3 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/sys/dev/pci/virtio_pci.c b/sys/dev/pci/virtio_pci.c
index 28087f1ac9d..398dc960f6d 100644
--- a/sys/dev/pci/virtio_pci.c
+++ b/sys/dev/pci/virtio_pci.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: virtio_pci.c,v 1.34 2023/07/05 18:11:08 patrick Exp $	*/
+/*	$OpenBSD: virtio_pci.c,v 1.35 2023/07/07 10:23:39 patrick Exp $	*/
 /*	$NetBSD: virtio.c,v 1.3 2011/11/02 23:05:52 njoly Exp $	*/
 
 /*
@@ -961,6 +961,13 @@ virtio_pci_setup_msix(struct virtio_pci_softc *sc, struct pci_attach_args *pa,
 	struct virtio_softc *vsc = &sc->sc_sc;
 	int i;
 
+	/* Shared needs config + queue */
+	if (shared && pci_intr_msix_count(pa) < 1 + 1)
+		return 1;
+	/* Per VQ needs config + N * queue */
+	if (!shared && pci_intr_msix_count(pa) < 1 + vsc->sc_nvqs)
+		return 1;
+
 	if (virtio_pci_msix_establish(sc, pa, 0, virtio_pci_config_intr, vsc))
 		return 1;
 	sc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI;
@@ -1059,10 +1066,9 @@ int
 virtio_pci_queue_intr(void *arg)
 {
 	struct virtqueue *vq = arg;
+	struct virtio_softc *vsc = vq->vq_owner;
 
-	if (vq->vq_done)
-		return (vq->vq_done)(vq);
-	return 0;
+	return virtio_check_vq(vsc, vq);
 }
 
 int
diff --git a/sys/dev/pv/virtio.c b/sys/dev/pv/virtio.c
index d085376d054..e9be61c0bbe 100644
--- a/sys/dev/pv/virtio.c
+++ b/sys/dev/pv/virtio.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: virtio.c,v 1.22 2023/04/20 19:28:31 jcs Exp $	*/
+/*	$OpenBSD: virtio.c,v 1.23 2023/07/07 10:23:39 patrick Exp $	*/
 /*	$NetBSD: virtio.c,v 1.3 2011/11/02 23:05:52 njoly Exp $	*/
 
 /*
@@ -218,26 +218,31 @@ vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot,
 int
 virtio_check_vqs(struct virtio_softc *sc)
 {
-	struct virtqueue *vq;
 	int i, r = 0;
 
 	/* going backwards is better for if_vio */
-	for (i = sc->sc_nvqs - 1; i >= 0; i--) {
-		vq = &sc->sc_vqs[i];
-		if (vq->vq_queued) {
-			vq->vq_queued = 0;
-			vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE);
-		}
-		vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD);
-		if (vq->vq_used_idx != vq->vq_used->idx) {
-			if (vq->vq_done)
-				r |= (vq->vq_done)(vq);
-		}
-	}
+	for (i = sc->sc_nvqs - 1; i >= 0; i--)
+		r |= virtio_check_vq(sc, &sc->sc_vqs[i]);
 
 	return r;
 }
 
+int
+virtio_check_vq(struct virtio_softc *sc, struct virtqueue *vq)
+{
+	if (vq->vq_queued) {
+		vq->vq_queued = 0;
+		vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE);
+	}
+	vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD);
+	if (vq->vq_used_idx != vq->vq_used->idx) {
+		if (vq->vq_done)
+			return (vq->vq_done)(vq);
+	}
+
+	return 0;
+}
+
 /*
  * Initialize vq structure.
  */
diff --git a/sys/dev/pv/virtiovar.h b/sys/dev/pv/virtiovar.h
index 9e3e9c9a27b..7f2aea701cd 100644
--- a/sys/dev/pv/virtiovar.h
+++ b/sys/dev/pv/virtiovar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: virtiovar.h,v 1.14 2019/05/26 15:22:31 sf Exp $	*/
+/*	$OpenBSD: virtiovar.h,v 1.15 2023/07/07 10:23:39 patrick Exp $	*/
 /*	$NetBSD: virtiovar.h,v 1.1 2011/10/30 12:12:21 hannken Exp $	*/
 
 /*
@@ -233,6 +233,7 @@ int virtio_dequeue_commit(struct virtqueue*, int);
 
 int virtio_intr(void *arg);
 int virtio_check_vqs(struct virtio_softc *);
+int virtio_check_vq(struct virtio_softc *, struct virtqueue *);
 void virtio_stop_vq_intr(struct virtio_softc *, struct virtqueue *);
 int virtio_start_vq_intr(struct virtio_softc *, struct virtqueue *);
 
-- 
2.20.1