Keep more information about the established LPI around, so that we can use

targeted invalidation through INV instead of flushing the whole cache through
INVALL.  Having this information enables us to send DISCARD, which clears the
mapping from the ITT.  This seems to be necessary to make the Hetzner VM's ITS
happy when we try to disestablish and re-establish an LPI.

This also moves the LPI table completely into agintc(4), as LPIs are global to
an agintc(4) and especially with multiple agintcmsi(4) they should be unique.

Tested by claudio@
ok kettenis@

diff --git a/sys/arch/arm64/dev/agintc.c b/sys/arch/arm64/dev/agintc.c
index 2714978..7f4a31a 100644 (file)
--- a/sys/arch/arm64/dev/agintc.c
+++ b/sys/arch/arm64/dev/agintc.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: agintc.c,v 1.51 2023/07/06 09:40:36 patrick Exp $ */
+/* $OpenBSD: agintc.c,v 1.52 2023/07/07 10:11:39 patrick Exp $ */
 /*
  * Copyright (c) 2007, 2009, 2011, 2017 Dale Rahn <drahn@dalerahn.com>
  * Copyright (c) 2018 Mark Kettenis <kettenis@openbsd.org>
@@ -148,10 +148,18 @@ struct agintc_mbi_range {
        void                    **mr_mbi;
 };
 
+struct agintc_lpi_info {
+       struct agintc_msi_softc *li_msic;
+       struct cpu_info         *li_ci;
+       uint32_t                 li_deviceid;
+       uint32_t                 li_eventid;
+       struct intrhand         *li_ih;
+};
+
 struct agintc_softc {
        struct simplebus_softc   sc_sbus;
        struct intrq            *sc_handler;
-       struct intrhand         **sc_lpi_handler;
+       struct agintc_lpi_info  **sc_lpi;
        bus_space_tag_t          sc_iot;
        bus_space_handle_t       sc_d_ioh;
        bus_space_handle_t      *sc_r_ioh;
@@ -257,7 +265,8 @@ int         agintc_ipi_nop(void *v);
 int            agintc_ipi_combined(void *);
 void           agintc_send_ipi(struct cpu_info *, int);
 
-void           agintc_msi_invall(void);
+void           agintc_msi_discard(struct agintc_lpi_info *);
+void           agintc_msi_inv(struct agintc_lpi_info *);
 
 const struct cfattach  agintc_ca = {
        sizeof (struct agintc_softc), agintc_match, agintc_attach
@@ -536,8 +545,8 @@ agintc_attach(struct device *parent, struct device *self, void *aux)
            sizeof(*sc->sc_handler), M_DEVBUF, M_ZERO | M_WAITOK);
        for (i = 0; i < nintr; i++)
                TAILQ_INIT(&sc->sc_handler[i].iq_list);
-       sc->sc_lpi_handler = mallocarray(sc->sc_nlpi,
-           sizeof(*sc->sc_lpi_handler), M_DEVBUF, M_ZERO | M_WAITOK);
+       sc->sc_lpi = mallocarray(sc->sc_nlpi,
+           sizeof(*sc->sc_lpi), M_DEVBUF, M_ZERO | M_WAITOK);
 
        /* set priority to IPL_HIGH until configure lowers to desired IPL */
        agintc_setipl(IPL_HIGH);
@@ -834,8 +843,11 @@ agintc_enable_wakeup(void)
        }
 
        for (irq = 0; irq < sc->sc_nlpi; irq++) {
-               ih = sc->sc_lpi_handler[irq];
-               if (ih == NULL || (ih->ih_flags & IPL_WAKEUP))
+               if (sc->sc_lpi[irq] == NULL)
+                       continue;
+               ih = sc->sc_lpi[irq]->li_ih;
+               KASSERT(ih != NULL);
+               if (ih->ih_flags & IPL_WAKEUP)
                        continue;
                prop = AGINTC_DMA_KVA(sc->sc_prop);
                prop[irq] &= ~GICR_PROP_ENABLE;
@@ -843,10 +855,9 @@ agintc_enable_wakeup(void)
                cpu_dcache_wb_range((vaddr_t)&prop[irq],
                    sizeof(*prop));
                __asm volatile("dsb sy");
+               /* Invalidate cache */
+               agintc_msi_inv(sc->sc_lpi[irq]);
        }
-
-       /* Invalidate cache. */
-       agintc_msi_invall();
 }
 
 void
@@ -874,8 +885,11 @@ agintc_disable_wakeup(void)
        }
 
        for (irq = 0; irq < sc->sc_nlpi; irq++) {
-               ih = sc->sc_lpi_handler[irq];
-               if (ih == NULL || (ih->ih_flags & IPL_WAKEUP))
+               if (sc->sc_lpi[irq] == NULL)
+                       continue;
+               ih = sc->sc_lpi[irq]->li_ih;
+               KASSERT(ih != NULL);
+               if (ih->ih_flags & IPL_WAKEUP)
                        continue;
                prop = AGINTC_DMA_KVA(sc->sc_prop);
                prop[irq] |= GICR_PROP_ENABLE;
@@ -883,10 +897,9 @@ agintc_disable_wakeup(void)
                cpu_dcache_wb_range((vaddr_t)&prop[irq],
                    sizeof(*prop));
                __asm volatile("dsb sy");
+               /* Invalidate cache */
+               agintc_msi_inv(sc->sc_lpi[irq]);
        }
-
-       /* Invalidate cache. */
-       agintc_msi_invall();
 }
 
 void
@@ -1144,10 +1157,11 @@ agintc_irq_handler(void *frame)
        }
 
        if (irq >= LPI_BASE) {
-               ih = sc->sc_lpi_handler[irq - LPI_BASE];
-               if (ih == NULL)
+               if (sc->sc_lpi[irq - LPI_BASE] == NULL)
                        return;
-               
+               ih = sc->sc_lpi[irq - LPI_BASE]->li_ih;
+               KASSERT(ih != NULL);
+
                s = agintc_splraise(ih->ih_ipl);
                intr_enable();
                agintc_run_handler(ih, frame, s);
@@ -1234,8 +1248,7 @@ agintc_intr_establish(int irqno, int type, int level, struct cpu_info *ci,
                }
                TAILQ_INSERT_TAIL(&sc->sc_handler[irqno].iq_list, ih, ih_list);
                sc->sc_handler[irqno].iq_ci = ci;
-       } else
-               sc->sc_lpi_handler[irqno - LPI_BASE] = ih;
+       }
 
        if (name != NULL)
                evcount_attach(&ih->ih_count, name, &ih->ih_irq);
@@ -1292,15 +1305,12 @@ agintc_intr_disestablish(void *cookie)
        } else {
                uint8_t *prop = AGINTC_DMA_KVA(sc->sc_prop);
 
-               sc->sc_lpi_handler[irqno - LPI_BASE] = NULL;
                prop[irqno - LPI_BASE] = 0;
 
                /* Make globally visible. */
                cpu_dcache_wb_range((vaddr_t)&prop[irqno - LPI_BASE],
                    sizeof(*prop));
                __asm volatile("dsb sy");
-
-               /* XXX: Invalidate cache? */
        }
 
        if (ih->ih_name != NULL)
@@ -1521,6 +1531,7 @@ struct gits_cmd {
 #define MAPTI  0x0a
 #define INV    0x0c
 #define INVALL 0x0d
+#define DISCARD 0x0f
 
 #define GITS_CMDQ_SIZE         (64 * 1024)
 #define GITS_CMDQ_NENTRIES     (GITS_CMDQ_SIZE / sizeof(struct gits_cmd))
@@ -1549,9 +1560,6 @@ struct agintc_msi_softc {
        bus_addr_t                      sc_msi_addr;
        int                             sc_msi_delta;
 
-       int                             sc_nlpi;
-       void                            **sc_lpi;
-
        struct agintc_dmamem            *sc_cmdq;
        uint16_t                        sc_cmdidx;
 
@@ -1631,10 +1639,6 @@ agintc_msi_attach(struct device *parent, struct device *self, void *aux)
        else
                sc->sc_cidbits = 16;
 
-       sc->sc_nlpi = agintc_sc->sc_nlpi;
-       sc->sc_lpi = mallocarray(sc->sc_nlpi, sizeof(void *), M_DEVBUF,
-           M_WAITOK|M_ZERO);
-
        /* Set up command queue. */
        sc->sc_cmdq = agintc_dmamem_alloc(sc->sc_dmat,
            GITS_CMDQ_SIZE, GITS_CMDQ_SIZE);
@@ -1802,9 +1806,6 @@ unmap:
        if (sc->sc_cmdq)
                agintc_dmamem_free(sc->sc_dmat, sc->sc_cmdq);
 
-       if (sc->sc_lpi)
-               free(sc->sc_lpi, M_DEVBUF, sc->sc_nlpi * sizeof(void *));
-
        bus_space_unmap(sc->sc_iot, sc->sc_ioh, faa->fa_reg[0].size);
 }
 
@@ -1878,25 +1879,53 @@ agintc_msi_find_device(struct agintc_msi_softc *sc, uint32_t deviceid)
 }
 
 void
-agintc_msi_invall(void)
+agintc_msi_discard(struct agintc_lpi_info *li)
 {
-       struct cfdriver *cd = &agintcmsi_cd;
        struct agintc_msi_softc *sc;
+       struct cpu_info *ci;
        struct gits_cmd cmd;
-       int i, j;
+       int hwcpu;
 
-       for (i = 0; i < cd->cd_ndevs; i++) {
-               if (cd->cd_devs[i] == NULL)
-                       continue;
-               sc = cd->cd_devs[i];
-               for (j = 0; j < ncpus; j++) {
-                       memset(&cmd, 0, sizeof(cmd));
-                       cmd.cmd = INVALL;
-                       cmd.dw2 = j;
-                       agintc_msi_send_cmd(sc, &cmd);
-                       agintc_msi_wait_cmd(sc);
-               }
-       }
+       sc = li->li_msic;
+       ci = li->li_ci;
+       hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid];
+
+       memset(&cmd, 0, sizeof(cmd));
+       cmd.cmd = DISCARD;
+       cmd.deviceid = li->li_deviceid;
+       cmd.eventid = li->li_eventid;
+       agintc_msi_send_cmd(sc, &cmd);
+
+       memset(&cmd, 0, sizeof(cmd));
+       cmd.cmd = SYNC;
+       cmd.dw2 = agintc_sc->sc_processor[hwcpu] << 16;
+       agintc_msi_send_cmd(sc, &cmd);
+       agintc_msi_wait_cmd(sc);
+}
+
+void
+agintc_msi_inv(struct agintc_lpi_info *li)
+{
+       struct agintc_msi_softc *sc;
+       struct cpu_info *ci;
+       struct gits_cmd cmd;
+       int hwcpu;
+
+       sc = li->li_msic;
+       ci = li->li_ci;
+       hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid];
+
+       memset(&cmd, 0, sizeof(cmd));
+       cmd.cmd = INV;
+       cmd.deviceid = li->li_deviceid;
+       cmd.eventid = li->li_eventid;
+       agintc_msi_send_cmd(sc, &cmd);
+
+       memset(&cmd, 0, sizeof(cmd));
+       cmd.cmd = SYNC;
+       cmd.dw2 = agintc_sc->sc_processor[hwcpu] << 16;
+       agintc_msi_send_cmd(sc, &cmd);
+       agintc_msi_wait_cmd(sc);
 }
 
 void *
@@ -1908,7 +1937,6 @@ agintc_intr_establish_msi(void *self, uint64_t *addr, uint64_t *data,
        struct gits_cmd cmd;
        uint32_t deviceid = *data;
        uint32_t eventid;
-       void *cookie;
        int i, hwcpu;
 
        if (ci == NULL)
@@ -1923,14 +1951,25 @@ agintc_intr_establish_msi(void *self, uint64_t *addr, uint64_t *data,
        if (eventid >= 32)
                return NULL;
 
-       for (i = 0; i < sc->sc_nlpi; i++) {
-               if (sc->sc_lpi[i] != NULL)
+       for (i = 0; i < agintc_sc->sc_nlpi; i++) {
+               if (agintc_sc->sc_lpi[i] != NULL)
                        continue;
 
-               cookie = agintc_intr_establish(LPI_BASE + i,
+               agintc_sc->sc_lpi[i] = malloc(sizeof(struct agintc_lpi_info),
+                   M_DEVBUF, M_WAITOK | M_ZERO);
+               agintc_sc->sc_lpi[i]->li_msic = sc;
+               agintc_sc->sc_lpi[i]->li_ci = ci;
+               agintc_sc->sc_lpi[i]->li_deviceid = deviceid;
+               agintc_sc->sc_lpi[i]->li_eventid = eventid;
+               agintc_sc->sc_lpi[i]->li_ih =
+                   agintc_intr_establish(LPI_BASE + i,
                    IST_EDGE_RISING, level, ci, func, arg, name);
-               if (cookie == NULL)
+               if (agintc_sc->sc_lpi[i]->li_ih == NULL) {
+                       free(agintc_sc->sc_lpi[i], M_DEVBUF,
+                           sizeof(struct agintc_lpi_info));
+                       agintc_sc->sc_lpi[i] = NULL;
                        return NULL;
+               }
 
                memset(&cmd, 0, sizeof(cmd));
                cmd.cmd = MAPTI;
@@ -1948,8 +1987,7 @@ agintc_intr_establish_msi(void *self, uint64_t *addr, uint64_t *data,
 
                *addr = sc->sc_msi_addr + deviceid * sc->sc_msi_delta;
                *data = eventid;
-               sc->sc_lpi[i] = cookie;
-               return &sc->sc_lpi[i];
+               return &agintc_sc->sc_lpi[i];
        }
 
        return NULL;
@@ -1958,17 +1996,22 @@ agintc_intr_establish_msi(void *self, uint64_t *addr, uint64_t *data,
 void
 agintc_intr_disestablish_msi(void *cookie)
 {
-       agintc_intr_disestablish(*(void **)cookie);
-       *(void **)cookie = NULL;
+       struct agintc_lpi_info *li = *(void **)cookie;
+
+       agintc_intr_disestablish(li->li_ih);
+       agintc_msi_discard(li);
+       agintc_msi_inv(li);
 
-       /* Invalidate cache. */
-       agintc_msi_invall();
+       free(li, M_DEVBUF, sizeof(*li));
+       *(void **)cookie = NULL;
 }
 
 void
 agintc_intr_barrier_msi(void *cookie)
 {
-       agintc_intr_barrier(*(void **)cookie);
+       struct agintc_lpi_info *li = *(void **)cookie;
+
+       agintc_intr_barrier(li->li_ih);
 }
 
 struct agintc_dmamem *