From e6a428abee99449cbc94a7ede74c7409b7b29614 Mon Sep 17 00:00:00 2001
From: dv <dv@openbsd.org>
Date: Mon, 15 Jan 2024 02:35:23 +0000
Subject: [PATCH] vio(4): poll device status after issuing device reset.

The virtio spec says a driver "should" wait for a device to report
a clear device status after performing a reset. In some hypervisors,
this doesn't matter as the vcpu's io instruction emulation and
virtio network device emulation happen serially in the same thread.
In hypervisors like vmd(8), device reset happens asynchronously and
the driver can't assume the device is ready.

This race condition results in mbuf pool corruption, causing panics.

Bug reported and reproduced by bluhm@. Root cause found and diff
from sf@. ok dv@ and committed on sf@'s behalf with his permission.
---
 sys/dev/fdt/virtio_mmio.c | 18 +++++++++++++-----
 sys/dev/pci/virtio_pci.c  | 26 ++++++++++++++++++++------
 2 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/sys/dev/fdt/virtio_mmio.c b/sys/dev/fdt/virtio_mmio.c
index 4f1e9eba9b7..4d92508f72e 100644
--- a/sys/dev/fdt/virtio_mmio.c
+++ b/sys/dev/fdt/virtio_mmio.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: virtio_mmio.c,v 1.11 2023/05/29 08:13:35 sf Exp $	*/
+/*	$OpenBSD: virtio_mmio.c,v 1.12 2024/01/15 02:35:23 dv Exp $	*/
 /*	$NetBSD: virtio.c,v 1.3 2011/11/02 23:05:52 njoly Exp $	*/
 
 /*
@@ -200,11 +200,19 @@ virtio_mmio_set_status(struct virtio_softc *vsc, int status)
 	struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc;
 	int old = 0;
 
-	if (status != 0)
+	if (status == 0) {
+		bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_STATUS,
+		    0);
+		while (bus_space_read_4(sc->sc_iot, sc->sc_ioh,
+		    VIRTIO_MMIO_STATUS) != 0) {
+			CPU_BUSY_CYCLE();
+		}
+	} else  {
 		old = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
-				       VIRTIO_MMIO_STATUS);
-	bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_STATUS,
-			  status|old);
+		    VIRTIO_MMIO_STATUS);
+		bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_STATUS,
+		    status|old);
+	}
 }
 
 int
diff --git a/sys/dev/pci/virtio_pci.c b/sys/dev/pci/virtio_pci.c
index 398dc960f6d..36a62c13bd1 100644
--- a/sys/dev/pci/virtio_pci.c
+++ b/sys/dev/pci/virtio_pci.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: virtio_pci.c,v 1.35 2023/07/07 10:23:39 patrick Exp $	*/
+/*	$OpenBSD: virtio_pci.c,v 1.36 2024/01/15 02:35:23 dv Exp $	*/
 /*	$NetBSD: virtio.c,v 1.3 2011/11/02 23:05:52 njoly Exp $	*/
 
 /*
@@ -282,15 +282,29 @@ virtio_pci_set_status(struct virtio_softc *vsc, int status)
 	int old = 0;
 
 	if (sc->sc_sc.sc_version_1) {
-		if (status != 0)
+		if (status == 0) {
+			CWRITE(sc, device_status, 0);
+			while (CREAD(sc, device_status) != 0) {
+				CPU_BUSY_CYCLE();
+			}
+		} else {
 			old = CREAD(sc, device_status);
-		CWRITE(sc, device_status, status|old);
+			CWRITE(sc, device_status, status|old);
+		}
 	} else {
-		if (status != 0)
+		if (status == 0) {
+			bus_space_write_1(sc->sc_iot, sc->sc_ioh,
+			    VIRTIO_CONFIG_DEVICE_STATUS, status|old);
+			while (bus_space_read_1(sc->sc_iot, sc->sc_ioh,
+			    VIRTIO_CONFIG_DEVICE_STATUS) != 0) {
+				CPU_BUSY_CYCLE();
+			}
+		} else {
 			old = bus_space_read_1(sc->sc_iot, sc->sc_ioh,
 			    VIRTIO_CONFIG_DEVICE_STATUS);
-		bus_space_write_1(sc->sc_iot, sc->sc_ioh,
-		    VIRTIO_CONFIG_DEVICE_STATUS, status|old);
+			bus_space_write_1(sc->sc_iot, sc->sc_ioh,
+			    VIRTIO_CONFIG_DEVICE_STATUS, status|old);
+		}
 	}
 }
 
-- 
2.20.1