From f83e6fb8171b1ac7c849f2be7e06799114174ffc Mon Sep 17 00:00:00 2001 From: patrick Date: Fri, 7 Jul 2023 10:11:39 +0000 Subject: [PATCH] Keep more information about the established LPI around, so that we can use targeted invalidation through INV instead of flushing the whole cache through INVALL. Having this information enables us to send DISCARD, which clears the mapping from the ITT. This seems to be necessary to make the Hetzner VM's ITS happy when we try to disestablish and re-establish an LPI. This also moves the LPI table completely into agintc(4), as LPIs are global to an agintc(4) and especially with multiple agintcmsi(4) they should be unique. Tested by claudio@ ok kettenis@ --- sys/arch/arm64/dev/agintc.c | 163 +++++++++++++++++++++++------------- 1 file changed, 103 insertions(+), 60 deletions(-) diff --git a/sys/arch/arm64/dev/agintc.c b/sys/arch/arm64/dev/agintc.c index 27149783a20..7f4a31aae45 100644 --- a/sys/arch/arm64/dev/agintc.c +++ b/sys/arch/arm64/dev/agintc.c @@ -1,4 +1,4 @@ -/* $OpenBSD: agintc.c,v 1.51 2023/07/06 09:40:36 patrick Exp $ */ +/* $OpenBSD: agintc.c,v 1.52 2023/07/07 10:11:39 patrick Exp $ */ /* * Copyright (c) 2007, 2009, 2011, 2017 Dale Rahn * Copyright (c) 2018 Mark Kettenis @@ -148,10 +148,18 @@ struct agintc_mbi_range { void **mr_mbi; }; +struct agintc_lpi_info { + struct agintc_msi_softc *li_msic; + struct cpu_info *li_ci; + uint32_t li_deviceid; + uint32_t li_eventid; + struct intrhand *li_ih; +}; + struct agintc_softc { struct simplebus_softc sc_sbus; struct intrq *sc_handler; - struct intrhand **sc_lpi_handler; + struct agintc_lpi_info **sc_lpi; bus_space_tag_t sc_iot; bus_space_handle_t sc_d_ioh; bus_space_handle_t *sc_r_ioh; @@ -257,7 +265,8 @@ int agintc_ipi_nop(void *v); int agintc_ipi_combined(void *); void agintc_send_ipi(struct cpu_info *, int); -void agintc_msi_invall(void); +void agintc_msi_discard(struct agintc_lpi_info *); +void agintc_msi_inv(struct agintc_lpi_info *); const struct cfattach agintc_ca = { sizeof (struct agintc_softc), agintc_match, agintc_attach @@ -536,8 +545,8 @@ agintc_attach(struct device *parent, struct device *self, void *aux) sizeof(*sc->sc_handler), M_DEVBUF, M_ZERO | M_WAITOK); for (i = 0; i < nintr; i++) TAILQ_INIT(&sc->sc_handler[i].iq_list); - sc->sc_lpi_handler = mallocarray(sc->sc_nlpi, - sizeof(*sc->sc_lpi_handler), M_DEVBUF, M_ZERO | M_WAITOK); + sc->sc_lpi = mallocarray(sc->sc_nlpi, + sizeof(*sc->sc_lpi), M_DEVBUF, M_ZERO | M_WAITOK); /* set priority to IPL_HIGH until configure lowers to desired IPL */ agintc_setipl(IPL_HIGH); @@ -834,8 +843,11 @@ agintc_enable_wakeup(void) } for (irq = 0; irq < sc->sc_nlpi; irq++) { - ih = sc->sc_lpi_handler[irq]; - if (ih == NULL || (ih->ih_flags & IPL_WAKEUP)) + if (sc->sc_lpi[irq] == NULL) + continue; + ih = sc->sc_lpi[irq]->li_ih; + KASSERT(ih != NULL); + if (ih->ih_flags & IPL_WAKEUP) continue; prop = AGINTC_DMA_KVA(sc->sc_prop); prop[irq] &= ~GICR_PROP_ENABLE; @@ -843,10 +855,9 @@ agintc_enable_wakeup(void) cpu_dcache_wb_range((vaddr_t)&prop[irq], sizeof(*prop)); __asm volatile("dsb sy"); + /* Invalidate cache */ + agintc_msi_inv(sc->sc_lpi[irq]); } - - /* Invalidate cache. */ - agintc_msi_invall(); } void @@ -874,8 +885,11 @@ agintc_disable_wakeup(void) } for (irq = 0; irq < sc->sc_nlpi; irq++) { - ih = sc->sc_lpi_handler[irq]; - if (ih == NULL || (ih->ih_flags & IPL_WAKEUP)) + if (sc->sc_lpi[irq] == NULL) + continue; + ih = sc->sc_lpi[irq]->li_ih; + KASSERT(ih != NULL); + if (ih->ih_flags & IPL_WAKEUP) continue; prop = AGINTC_DMA_KVA(sc->sc_prop); prop[irq] |= GICR_PROP_ENABLE; @@ -883,10 +897,9 @@ agintc_disable_wakeup(void) cpu_dcache_wb_range((vaddr_t)&prop[irq], sizeof(*prop)); __asm volatile("dsb sy"); + /* Invalidate cache */ + agintc_msi_inv(sc->sc_lpi[irq]); } - - /* Invalidate cache. */ - agintc_msi_invall(); } void @@ -1144,10 +1157,11 @@ agintc_irq_handler(void *frame) } if (irq >= LPI_BASE) { - ih = sc->sc_lpi_handler[irq - LPI_BASE]; - if (ih == NULL) + if (sc->sc_lpi[irq - LPI_BASE] == NULL) return; - + ih = sc->sc_lpi[irq - LPI_BASE]->li_ih; + KASSERT(ih != NULL); + s = agintc_splraise(ih->ih_ipl); intr_enable(); agintc_run_handler(ih, frame, s); @@ -1234,8 +1248,7 @@ agintc_intr_establish(int irqno, int type, int level, struct cpu_info *ci, } TAILQ_INSERT_TAIL(&sc->sc_handler[irqno].iq_list, ih, ih_list); sc->sc_handler[irqno].iq_ci = ci; - } else - sc->sc_lpi_handler[irqno - LPI_BASE] = ih; + } if (name != NULL) evcount_attach(&ih->ih_count, name, &ih->ih_irq); @@ -1292,15 +1305,12 @@ agintc_intr_disestablish(void *cookie) } else { uint8_t *prop = AGINTC_DMA_KVA(sc->sc_prop); - sc->sc_lpi_handler[irqno - LPI_BASE] = NULL; prop[irqno - LPI_BASE] = 0; /* Make globally visible. */ cpu_dcache_wb_range((vaddr_t)&prop[irqno - LPI_BASE], sizeof(*prop)); __asm volatile("dsb sy"); - - /* XXX: Invalidate cache? */ } if (ih->ih_name != NULL) @@ -1521,6 +1531,7 @@ struct gits_cmd { #define MAPTI 0x0a #define INV 0x0c #define INVALL 0x0d +#define DISCARD 0x0f #define GITS_CMDQ_SIZE (64 * 1024) #define GITS_CMDQ_NENTRIES (GITS_CMDQ_SIZE / sizeof(struct gits_cmd)) @@ -1549,9 +1560,6 @@ struct agintc_msi_softc { bus_addr_t sc_msi_addr; int sc_msi_delta; - int sc_nlpi; - void **sc_lpi; - struct agintc_dmamem *sc_cmdq; uint16_t sc_cmdidx; @@ -1631,10 +1639,6 @@ agintc_msi_attach(struct device *parent, struct device *self, void *aux) else sc->sc_cidbits = 16; - sc->sc_nlpi = agintc_sc->sc_nlpi; - sc->sc_lpi = mallocarray(sc->sc_nlpi, sizeof(void *), M_DEVBUF, - M_WAITOK|M_ZERO); - /* Set up command queue. */ sc->sc_cmdq = agintc_dmamem_alloc(sc->sc_dmat, GITS_CMDQ_SIZE, GITS_CMDQ_SIZE); @@ -1802,9 +1806,6 @@ unmap: if (sc->sc_cmdq) agintc_dmamem_free(sc->sc_dmat, sc->sc_cmdq); - if (sc->sc_lpi) - free(sc->sc_lpi, M_DEVBUF, sc->sc_nlpi * sizeof(void *)); - bus_space_unmap(sc->sc_iot, sc->sc_ioh, faa->fa_reg[0].size); } @@ -1878,25 +1879,53 @@ agintc_msi_find_device(struct agintc_msi_softc *sc, uint32_t deviceid) } void -agintc_msi_invall(void) +agintc_msi_discard(struct agintc_lpi_info *li) { - struct cfdriver *cd = &agintcmsi_cd; struct agintc_msi_softc *sc; + struct cpu_info *ci; struct gits_cmd cmd; - int i, j; + int hwcpu; - for (i = 0; i < cd->cd_ndevs; i++) { - if (cd->cd_devs[i] == NULL) - continue; - sc = cd->cd_devs[i]; - for (j = 0; j < ncpus; j++) { - memset(&cmd, 0, sizeof(cmd)); - cmd.cmd = INVALL; - cmd.dw2 = j; - agintc_msi_send_cmd(sc, &cmd); - agintc_msi_wait_cmd(sc); - } - } + sc = li->li_msic; + ci = li->li_ci; + hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid]; + + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = DISCARD; + cmd.deviceid = li->li_deviceid; + cmd.eventid = li->li_eventid; + agintc_msi_send_cmd(sc, &cmd); + + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = SYNC; + cmd.dw2 = agintc_sc->sc_processor[hwcpu] << 16; + agintc_msi_send_cmd(sc, &cmd); + agintc_msi_wait_cmd(sc); +} + +void +agintc_msi_inv(struct agintc_lpi_info *li) +{ + struct agintc_msi_softc *sc; + struct cpu_info *ci; + struct gits_cmd cmd; + int hwcpu; + + sc = li->li_msic; + ci = li->li_ci; + hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid]; + + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = INV; + cmd.deviceid = li->li_deviceid; + cmd.eventid = li->li_eventid; + agintc_msi_send_cmd(sc, &cmd); + + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = SYNC; + cmd.dw2 = agintc_sc->sc_processor[hwcpu] << 16; + agintc_msi_send_cmd(sc, &cmd); + agintc_msi_wait_cmd(sc); } void * @@ -1908,7 +1937,6 @@ agintc_intr_establish_msi(void *self, uint64_t *addr, uint64_t *data, struct gits_cmd cmd; uint32_t deviceid = *data; uint32_t eventid; - void *cookie; int i, hwcpu; if (ci == NULL) @@ -1923,14 +1951,25 @@ agintc_intr_establish_msi(void *self, uint64_t *addr, uint64_t *data, if (eventid >= 32) return NULL; - for (i = 0; i < sc->sc_nlpi; i++) { - if (sc->sc_lpi[i] != NULL) + for (i = 0; i < agintc_sc->sc_nlpi; i++) { + if (agintc_sc->sc_lpi[i] != NULL) continue; - cookie = agintc_intr_establish(LPI_BASE + i, + agintc_sc->sc_lpi[i] = malloc(sizeof(struct agintc_lpi_info), + M_DEVBUF, M_WAITOK | M_ZERO); + agintc_sc->sc_lpi[i]->li_msic = sc; + agintc_sc->sc_lpi[i]->li_ci = ci; + agintc_sc->sc_lpi[i]->li_deviceid = deviceid; + agintc_sc->sc_lpi[i]->li_eventid = eventid; + agintc_sc->sc_lpi[i]->li_ih = + agintc_intr_establish(LPI_BASE + i, IST_EDGE_RISING, level, ci, func, arg, name); - if (cookie == NULL) + if (agintc_sc->sc_lpi[i]->li_ih == NULL) { + free(agintc_sc->sc_lpi[i], M_DEVBUF, + sizeof(struct agintc_lpi_info)); + agintc_sc->sc_lpi[i] = NULL; return NULL; + } memset(&cmd, 0, sizeof(cmd)); cmd.cmd = MAPTI; @@ -1948,8 +1987,7 @@ agintc_intr_establish_msi(void *self, uint64_t *addr, uint64_t *data, *addr = sc->sc_msi_addr + deviceid * sc->sc_msi_delta; *data = eventid; - sc->sc_lpi[i] = cookie; - return &sc->sc_lpi[i]; + return &agintc_sc->sc_lpi[i]; } return NULL; @@ -1958,17 +1996,22 @@ agintc_intr_establish_msi(void *self, uint64_t *addr, uint64_t *data, void agintc_intr_disestablish_msi(void *cookie) { - agintc_intr_disestablish(*(void **)cookie); - *(void **)cookie = NULL; + struct agintc_lpi_info *li = *(void **)cookie; + + agintc_intr_disestablish(li->li_ih); + agintc_msi_discard(li); + agintc_msi_inv(li); - /* Invalidate cache. */ - agintc_msi_invall(); + free(li, M_DEVBUF, sizeof(*li)); + *(void **)cookie = NULL; } void agintc_intr_barrier_msi(void *cookie) { - agintc_intr_barrier(*(void **)cookie); + struct agintc_lpi_info *li = *(void **)cookie; + + agintc_intr_barrier(li->li_ih); } struct agintc_dmamem * -- 2.20.1