From: kettenis Date: Mon, 30 Jul 2018 10:56:00 +0000 (+0000) Subject: Add support for the GIC v3 ITS and use it to implement MSI support for X-Git-Url: http://artulab.com/gitweb/?a=commitdiff_plain;h=f0a2ac1908bc1d3309db7af62ab8d3bd41fe8cc3;p=openbsd Add support for the GIC v3 ITS and use it to implement MSI support for rkpcie(4). ok patrick@ --- diff --git a/sys/arch/arm64/arm64/intr.c b/sys/arch/arm64/arm64/intr.c index 574e3cc0d0f..96e27963e5c 100644 --- a/sys/arch/arm64/arm64/intr.c +++ b/sys/arch/arm64/arm64/intr.c @@ -1,4 +1,4 @@ -/* $OpenBSD: intr.c,v 1.11 2018/04/09 18:35:13 kettenis Exp $ */ +/* $OpenBSD: intr.c,v 1.12 2018/07/30 10:56:00 kettenis Exp $ */ /* * Copyright (c) 2011 Dale Rahn * @@ -27,7 +27,7 @@ #include uint32_t arm_intr_get_parent(int); -uint32_t arm_intr_msi_get_parent(int); +uint32_t arm_intr_map_msi(int, uint64_t *); void *arm_intr_prereg_establish_fdt(void *, int *, int, int (*)(void *), void *, char *); @@ -82,15 +82,67 @@ arm_intr_get_parent(int node) } uint32_t -arm_intr_msi_get_parent(int node) +arm_intr_map_msi(int node, uint64_t *data) { + uint64_t msi_base; uint32_t phandle = 0; + uint32_t *cell; + uint32_t *map; + uint32_t mask, rid_base, rid; + int i, len, length, mcells, ncells; - while (node && !phandle) { - phandle = OF_getpropint(node, "msi-parent", 0); - node = OF_parent(node); + len = OF_getproplen(node, "msi-map"); + if (len <= 0) { + while (node && !phandle) { + phandle = OF_getpropint(node, "msi-parent", 0); + node = OF_parent(node); + } + + return phandle; + } + + map = malloc(len, M_TEMP, M_WAITOK); + OF_getpropintarray(node, "msi-map", map, len); + + mask = OF_getpropint(node, "msi-map-mask", 0xffff); + rid = *data & mask; + + cell = map; + ncells = len / sizeof(uint32_t); + while (ncells > 1) { + node = OF_getnodebyphandle(cell[1]); + if (node == 0) + goto out; + + /* + * Some device trees (e.g. those for the Rockchip + * RK3399 boards) are missing a #msi-cells property. + * Assume the msi-specifier uses a single cell in that + * case. + */ + mcells = OF_getpropint(node, "#msi-cells", 1); + if (ncells < mcells + 3) + goto out; + + rid_base = cell[0]; + length = cell[2 + mcells]; + msi_base = cell[2]; + for (i = 1; i < mcells; i++) { + msi_base <<= 32; + msi_base |= cell[2 + i]; + } + if (rid >= rid_base && rid < rid_base + length) { + *data = msi_base + (rid - rid_base); + phandle = cell[1]; + break; + } + + cell += (3 + mcells); + ncells -= (3 + mcells); } +out: + free(map, M_TEMP, len); return phandle; } @@ -316,7 +368,7 @@ arm_intr_establish_fdt_imap(int node, int *reg, int nreg, int level, struct arm_intr_handle *ih; uint32_t *cell; uint32_t map_mask[4], *map; - int i, len, acells, ncells; + int len, acells, ncells; void *val = NULL; if (nreg != sizeof(map_mask)) @@ -335,7 +387,7 @@ arm_intr_establish_fdt_imap(int node, int *reg, int nreg, int level, cell = map; ncells = len / sizeof(uint32_t); - for (i = 0; ncells > 0; i++) { + while (ncells > 5) { LIST_FOREACH(ic, &interrupt_controllers, ic_list) { if (ic->ic_phandle == cell[4]) break; @@ -382,7 +434,7 @@ arm_intr_establish_fdt_msi(int node, uint64_t *addr, uint64_t *data, uint32_t phandle; void *val = NULL; - phandle = arm_intr_msi_get_parent(node); + phandle = arm_intr_map_msi(node, data); LIST_FOREACH(ic, &interrupt_controllers, ic_list) { if (ic->ic_phandle == phandle) break; diff --git a/sys/arch/arm64/conf/GENERIC b/sys/arch/arm64/conf/GENERIC index 517df52fa0a..706cd0740ab 100644 --- a/sys/arch/arm64/conf/GENERIC +++ b/sys/arch/arm64/conf/GENERIC @@ -1,4 +1,4 @@ -# $OpenBSD: GENERIC,v 1.79 2018/07/30 08:14:45 patrick Exp $ +# $OpenBSD: GENERIC,v 1.80 2018/07/30 10:56:00 kettenis Exp $ # # GENERIC machine description file # @@ -58,7 +58,8 @@ uk* at scsibus? ampintc* at fdt? early 1 ampintcmsi* at fdt? early 1 -agintc* at fdt? +agintc* at fdt? early 1 +agintcmsi* at fdt? early 1 agtimer* at fdt? ahci* at fdt? dwge* at fdt? diff --git a/sys/arch/arm64/conf/RAMDISK b/sys/arch/arm64/conf/RAMDISK index 4169488260d..b8e67410b61 100644 --- a/sys/arch/arm64/conf/RAMDISK +++ b/sys/arch/arm64/conf/RAMDISK @@ -1,4 +1,4 @@ -# $OpenBSD: RAMDISK,v 1.64 2018/07/05 19:25:38 kettenis Exp $ +# $OpenBSD: RAMDISK,v 1.65 2018/07/30 10:56:00 kettenis Exp $ # # GENERIC machine description file # @@ -68,7 +68,8 @@ uk* at scsibus? ampintc* at fdt? early 1 ampintcmsi* at fdt? early 1 -agintc* at fdt? +agintc* at fdt? early 1 +agintcmsi* at fdt? early 1 agtimer* at fdt? ahci* at fdt? dwge* at fdt? diff --git a/sys/arch/arm64/conf/files.arm64 b/sys/arch/arm64/conf/files.arm64 index b2f24a2cc15..f67c9d9aec5 100644 --- a/sys/arch/arm64/conf/files.arm64 +++ b/sys/arch/arm64/conf/files.arm64 @@ -1,4 +1,4 @@ -# $OpenBSD: files.arm64,v 1.25 2018/07/05 19:25:38 kettenis Exp $ +# $OpenBSD: files.arm64,v 1.26 2018/07/30 10:56:00 kettenis Exp $ maxpartitions 16 maxusers 2 8 64 @@ -117,9 +117,11 @@ device ampintcmsi attach ampintcmsi at fdt file arch/arm64/dev/ampintc.c ampintc | ampintcmsi -device agintc +device agintc: fdt attach agintc at fdt -file arch/arm64/dev/agintc.c agintc +device agintcmsi +attach agintcmsi at fdt +file arch/arm64/dev/agintc.c agintc | agintcmsi device agtimer attach agtimer at fdt diff --git a/sys/arch/arm64/dev/agintc.c b/sys/arch/arm64/dev/agintc.c index 0dffaa4f37a..32e99653470 100644 --- a/sys/arch/arm64/dev/agintc.c +++ b/sys/arch/arm64/dev/agintc.c @@ -1,6 +1,7 @@ -/* $OpenBSD: agintc.c,v 1.8 2018/03/30 16:55:20 patrick Exp $ */ +/* $OpenBSD: agintc.c,v 1.9 2018/07/30 10:56:00 kettenis Exp $ */ /* * Copyright (c) 2007, 2009, 2011, 2017 Dale Rahn + * Copyright (c) 2018 Mark Kettenis * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -33,6 +34,8 @@ #include #include +#include + #define ICC_PMR s3_0_c4_c6_0 #define ICC_IAR0 s3_0_c12_c8_0 #define ICC_EOIR0 s3_0_c12_c8_1 @@ -65,6 +68,7 @@ #define GICD_CTRL_EnableGrp1A (1 << 1) #define GICD_CTRL_ARE_NS (1 << 4) #define GICD_TYPER 0x0004 +#define GICD_TYPER_LPIS (1 << 16) #define GICD_TYPER_ITLINE_M 0xf #define GICD_IIDR 0x0008 #define GICD_ISENABLER(i) (0x0100 + (IRQ_TO_REG32(i) * 4)) @@ -80,6 +84,7 @@ /* redistributor registers */ #define GICR_CTLR 0x00000 #define GICR_CTLR_RWP ((1U << 31) | (1 << 3)) +#define GICR_CTLR_ENABLE_LPIS (1 << 0) #define GICR_IIDR 0x00004 #define GICR_TYPER 0x00008 #define GICR_TYPER_LAST (1 << 4) @@ -89,6 +94,13 @@ #define GICR_WAKER_CHILDRENASLEEP (1 << 2) #define GICR_WAKER_PROCESSORSLEEP (1 << 1) #define GICR_WAKER_X0 (1 << 0) +#define GICR_PROPBASER 0x00070 +#define GICR_PROPBASER_ISH (1ULL << 10) +#define GICR_PROPBASER_IC_NORM_NC (1ULL << 7) +#define GICR_PENDBASER 0x00078 +#define GICR_PENDBASER_PTZ (1ULL << 62) +#define GICR_PENDBASER_ISH (1ULL << 10) +#define GICR_PENDBASER_IC_NORM_NC (1ULL << 7) #define GICR_IGROUP0 0x10080 #define GICR_ISENABLE0 0x10100 #define GICR_ICENABLE0 0x10180 @@ -100,6 +112,14 @@ #define GICR_ICFGR0 0x10c00 #define GICR_ICFGR1 0x10c04 +#define GICR_PROP_SIZE (64 * 1024) +#define GICR_PROP_ENABLE (1 << 0) +#define GICR_PEND_SIZE (64 * 1024) + +#define PPI_BASE 16 +#define SPI_BASE 32 +#define LPI_BASE 8192 + #define IRQ_TO_REG32(i) (((i) >> 5) & 0x7) #define IRQ_TO_REG32BIT(i) ((i) & 0x1f) @@ -118,18 +138,23 @@ #define MAX_CORES 16 struct agintc_softc { - struct device sc_dev; - struct intrq *sc_agintc_handler; + struct simplebus_softc sc_sbus; + struct intrq *sc_handler; + struct intrhand **sc_lpi_handler; bus_space_tag_t sc_iot; bus_space_handle_t sc_d_ioh; bus_space_handle_t sc_r_ioh[MAX_CORES]; bus_space_handle_t sc_redist_base; + bus_dma_tag_t sc_dmat; uint64_t sc_affinity[MAX_CORES]; int sc_cpuremap[MAX_CORES]; /* bsd to redist */ int sc_nintr; + int sc_nlpi; struct evcount sc_spur; int sc_ncells; int sc_num_redist; + struct agintc_dmamem *sc_prop; + struct agintc_dmamem *sc_pend; struct interrupt_controller sc_ic; int sc_ipi_num[2]; /* id for NOP and DDB ipi */ int sc_ipi_reason[MAX_CORES]; /* NOP or DDB caused */ @@ -156,6 +181,22 @@ struct intrq { int iq_route; }; +struct agintc_dmamem { + bus_dmamap_t adm_map; + bus_dma_segment_t adm_seg; + size_t adm_size; + caddr_t adm_kva; +}; + +#define AGINTC_DMA_MAP(_adm) ((_adm)->adm_map) +#define AGINTC_DMA_LEN(_adm) ((_adm)->adm_size) +#define AGINTC_DMA_DVA(_adm) ((_adm)->adm_map->dm_segs[0].ds_addr) +#define AGINTC_DMA_KVA(_adm) ((void *)(_adm)->adm_kva) + +struct agintc_dmamem *agintc_dmamem_alloc(bus_dma_tag_t, bus_size_t, + bus_size_t); +void agintc_dmamem_free(bus_dma_tag_t, struct agintc_dmamem *); + int agintc_match(struct device *, void *, void *); void agintc_attach(struct device *, struct device *, void *); void agintc_cpuinit(void); @@ -226,6 +267,7 @@ agintc_attach(struct device *parent, struct device *self, void *aux) { struct agintc_softc *sc = (struct agintc_softc *)self; struct fdt_attach_args *faa = aux; + uint32_t typer; int i, j, nintr; int psw; int offset, nredist; @@ -238,6 +280,7 @@ agintc_attach(struct device *parent, struct device *self, void *aux) arm_init_smask(); sc->sc_iot = faa->fa_iot; + sc->sc_dmat = faa->fa_dmat; /* First row: distributor */ if (bus_space_map(sc->sc_iot, faa->fa_reg[0].addr, @@ -249,14 +292,33 @@ agintc_attach(struct device *parent, struct device *self, void *aux) faa->fa_reg[1].size, 0, &sc->sc_redist_base)) panic("%s: ICP bus_space_map failed!", __func__); + typer = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_TYPER); + + if (typer & GICD_TYPER_LPIS) { + /* Allocate redistributor tables */ + sc->sc_prop = agintc_dmamem_alloc(sc->sc_dmat, + GICR_PROP_SIZE, GICR_PROP_SIZE); + if (sc->sc_prop == NULL) { + printf(": can't alloc LPI config table\n"); + goto unmap; + } + sc->sc_pend = agintc_dmamem_alloc(sc->sc_dmat, + GICR_PEND_SIZE, GICR_PEND_SIZE); + if (sc->sc_prop == NULL) { + printf(": can't alloc LPI pending table\n"); + goto unmap; + } + + /* Minimum number of LPIs supported by any implementation. */ + sc->sc_nlpi = 8192; + } + evcount_attach(&sc->sc_spur, "irq1023/spur", NULL); __asm volatile("msr "STR(ICC_SRE_EL1)", %x0" : : "r" (ICC_SRE_EL1_EN)); __isb(); - nintr = 32 * (bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_TYPER) & - GICD_TYPER_ITLINE_M); - + nintr = 32 * (typer & GICD_TYPER_ITLINE_M); nintr += 32; /* ICD_ICTR + 1, irq 0-31 is SGI, 32+ is PPI */ sc->sc_nintr = nintr; @@ -285,6 +347,21 @@ agintc_attach(struct device *parent, struct device *self, void *aux) sc->sc_r_ioh[nredist]); #endif + if (sc->sc_nlpi > 0) { + bus_space_write_8(sc->sc_iot, sc->sc_redist_base, + offset + GICR_PROPBASER, + AGINTC_DMA_DVA(sc->sc_prop) | + GICR_PROPBASER_ISH | GICR_PROPBASER_IC_NORM_NC | + fls(LPI_BASE + sc->sc_nlpi - 1) - 1); + bus_space_write_8(sc->sc_iot, sc->sc_redist_base, + offset + GICR_PENDBASER, + AGINTC_DMA_DVA(sc->sc_pend) | + GICR_PENDBASER_ISH | GICR_PENDBASER_IC_NORM_NC | + GICR_PENDBASER_PTZ); + bus_space_write_4(sc->sc_iot, sc->sc_redist_base, + offset + GICR_CTLR, GICR_CTLR_ENABLE_LPIS); + } + offset += sz; if (typer & GICR_TYPER_LAST) { @@ -320,10 +397,12 @@ agintc_attach(struct device *parent, struct device *self, void *aux) agintc_cpuinit(); - sc->sc_agintc_handler = mallocarray(nintr, - sizeof(*sc->sc_agintc_handler), M_DEVBUF, M_ZERO | M_NOWAIT); + sc->sc_handler = mallocarray(nintr, + sizeof(*sc->sc_handler), M_DEVBUF, M_ZERO | M_WAITOK); for (i = 0; i < nintr; i++) - TAILQ_INIT(&sc->sc_agintc_handler[i].iq_list); + TAILQ_INIT(&sc->sc_handler[i].iq_list); + sc->sc_lpi_handler = mallocarray(sc->sc_nlpi, + sizeof(*sc->sc_lpi_handler), M_DEVBUF, M_ZERO | M_WAITOK); /* set priority to IPL_HIGH until configure lowers to desired IPL */ agintc_setipl(IPL_HIGH); @@ -406,8 +485,6 @@ agintc_attach(struct device *parent, struct device *self, void *aux) intr_send_ipi_func = agintc_send_ipi; #endif - printf("\n"); - sc->sc_ic.ic_node = faa->fa_node; sc->sc_ic.ic_cookie = self; sc->sc_ic.ic_establish = agintc_intr_establish_fdt; @@ -417,6 +494,20 @@ agintc_attach(struct device *parent, struct device *self, void *aux) arm_intr_register_fdt(&sc->sc_ic); restore_interrupts(psw); + + /* Attach ITS. */ + simplebus_attach(parent, &sc->sc_sbus.sc_dev, faa); + + return; + +unmap: + if (sc->sc_pend) + agintc_dmamem_free(sc->sc_dmat, sc->sc_pend); + if (sc->sc_prop) + agintc_dmamem_free(sc->sc_dmat, sc->sc_prop); + + bus_space_unmap(sc->sc_iot, sc->sc_redist_base, faa->fa_reg[1].size); + bus_space_unmap(sc->sc_iot, sc->sc_d_ioh, faa->fa_reg[0].size); } /* Initialize redistributors on each core. */ @@ -503,7 +594,7 @@ agintc_set_priority(struct agintc_softc *sc, int irq, int pri) * also low values are higher priority thus NIPL - pri */ prival = 0x80 | ((NIPL - pri) << 3); - if (irq >= 32) { + if (irq >= SPI_BASE) { bus_space_write_1(sc->sc_iot, sc->sc_d_ioh, GICD_IPRIORITYR(irq), prival); } else { @@ -597,7 +688,7 @@ agintc_calc_irq(struct agintc_softc *sc, int irq) int max = IPL_NONE; int min = IPL_HIGH; - TAILQ_FOREACH(ih, &sc->sc_agintc_handler[irq].iq_list, ih_list) { + TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) { if (ih->ih_ipl > max) max = ih->ih_ipl; @@ -605,9 +696,9 @@ agintc_calc_irq(struct agintc_softc *sc, int irq) min = ih->ih_ipl; } - if (sc->sc_agintc_handler[irq].iq_irq == max) + if (sc->sc_handler[irq].iq_irq == max) return; - sc->sc_agintc_handler[irq].iq_irq = max; + sc->sc_handler[irq].iq_irq = max; if (max == IPL_NONE) min = IPL_NONE; @@ -687,7 +778,7 @@ agintc_route_irq(void *v, int enable, struct cpu_info *ci) if (enable) { agintc_set_priority(sc, ih->ih_irq, - sc->sc_agintc_handler[ih->ih_irq].iq_irq); + sc->sc_handler[ih->ih_irq].iq_irq); agintc_route(sc, ih->ih_irq, IRQ_ENABLE, ci); agintc_intr_enable(sc, ih->ih_irq); } @@ -711,14 +802,48 @@ agintc_route(struct agintc_softc *sc, int irq, int enable, struct cpu_info *ci) } } +void +agintc_run_handler(struct intrhand *ih, void *frame, int s) +{ + void *arg; + int handled; + +#ifdef MULTIPROCESSOR + int need_lock; + + if (ih->ih_flags & IPL_MPSAFE) + need_lock = 0; + else + need_lock = s < IPL_SCHED; + + if (need_lock) + KERNEL_LOCK(); +#endif + + if (ih->ih_arg != 0) + arg = ih->ih_arg; + else + arg = frame; + + enable_interrupts(); + handled = ih->ih_func(arg); + disable_interrupts(); + if (handled) + ih->ih_count.ec_count++; + +#ifdef MULTIPROCESSOR + if (need_lock) + KERNEL_UNLOCK(); +#endif +} + void agintc_irq_handler(void *frame) { struct cpu_info *ci = curcpu(); struct agintc_softc *sc = agintc_sc; struct intrhand *ih; - void *arg; - int irq, pri, s, handled; + int irq, pri, s; ci->ci_idepth++; irq = agintc_iack(); @@ -743,41 +868,32 @@ agintc_irq_handler(void *frame) return; } - if (irq >= sc->sc_nintr) { + if ((irq >= sc->sc_nintr && irq < LPI_BASE) || + irq >= LPI_BASE + sc->sc_nlpi) { ci->ci_idepth--; return; } - pri = sc->sc_agintc_handler[irq].iq_irq; - s = agintc_splraise(pri); - TAILQ_FOREACH(ih, &sc->sc_agintc_handler[irq].iq_list, ih_list) { -#ifdef MULTIPROCESSOR - int need_lock; - - if (ih->ih_flags & IPL_MPSAFE) - need_lock = 0; - else - need_lock = s < IPL_SCHED; - - if (need_lock) - KERNEL_LOCK(); -#endif - - if (ih->ih_arg != 0) - arg = ih->ih_arg; - else - arg = frame; + if (irq >= LPI_BASE) { + ih = sc->sc_lpi_handler[irq - LPI_BASE]; + if (ih == NULL) { + ci->ci_idepth--; + return; + } + + s = agintc_splraise(ih->ih_ipl); + agintc_run_handler(ih, frame, s); + agintc_eoi(irq); - enable_interrupts(); - handled = ih->ih_func(arg); - disable_interrupts(); - if (handled) - ih->ih_count.ec_count++; + agintc_splx(s); + ci->ci_idepth--; + return; + } -#ifdef MULTIPROCESSOR - if (need_lock) - KERNEL_UNLOCK(); -#endif + pri = sc->sc_handler[irq].iq_irq; + s = agintc_splraise(pri); + TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) { + agintc_run_handler(ih, frame, s); } agintc_eoi(irq); @@ -797,11 +913,11 @@ agintc_intr_establish_fdt(void *cookie, int *cell, int level, /* 1st cell contains type: 0 SPI (32-X), 1 PPI (16-31) */ if (cell[0] == 0) - irq += 32; + irq += SPI_BASE; else if (cell[0] == 1) - irq += 16; + irq += PPI_BASE; else - panic("%s: bogus interrupt type", sc->sc_dev.dv_xname); + panic("%s: bogus interrupt type", sc->sc_sbus.sc_dev.dv_xname); return agintc_intr_establish(irq, level, func, arg, name); } @@ -811,10 +927,12 @@ agintc_intr_establish(int irqno, int level, int (*func)(void *), void *arg, char *name) { struct agintc_softc *sc = agintc_sc; + uint8_t *prop = AGINTC_DMA_KVA(sc->sc_prop); struct intrhand *ih; int psw; - if (irqno < 0 || irqno >= sc->sc_nintr) + if (irqno < 0 || (irqno >= sc->sc_nintr && irqno < LPI_BASE) || + irqno >= LPI_BASE + sc->sc_nlpi) panic("agintc_intr_establish: bogus irqnumber %d: %s", irqno, name); @@ -828,7 +946,10 @@ agintc_intr_establish(int irqno, int level, int (*func)(void *), psw = disable_interrupts(); - TAILQ_INSERT_TAIL(&sc->sc_agintc_handler[irqno].iq_list, ih, ih_list); + if (irqno < LPI_BASE) + TAILQ_INSERT_TAIL(&sc->sc_handler[irqno].iq_list, ih, ih_list); + else + sc->sc_lpi_handler[irqno - LPI_BASE] = ih; if (name != NULL) evcount_attach(&ih->ih_count, name, &ih->ih_irq); @@ -837,7 +958,13 @@ agintc_intr_establish(int irqno, int level, int (*func)(void *), printf("%s: irq %d level %d [%s]\n", __func__, irqno, level, name); #endif - agintc_calc_irq(sc, irqno); + if (irqno < LPI_BASE) { + agintc_calc_irq(sc, irqno); + } else { + prop[irqno - LPI_BASE] = + 0x80 | ((NIPL - level) << 3) | GICR_PROP_ENABLE; + __asm volatile("dsb sy"); /* make globally visible */ + } restore_interrupts(psw); return (ih); @@ -853,7 +980,7 @@ agintc_intr_disestablish(void *cookie) psw = disable_interrupts(); - TAILQ_REMOVE(&sc->sc_agintc_handler[irqno].iq_list, ih, ih_list); + TAILQ_REMOVE(&sc->sc_handler[irqno].iq_list, ih, ih_list); if (ih->ih_name != NULL) evcount_detach(&ih->ih_count); @@ -963,3 +1090,375 @@ agintc_send_ipi(struct cpu_info *ci, int id) __asm volatile ("msr " STR(ICC_SGI1R)", %x0" ::"r"(sendmask)); } #endif + +/* + * GICv3 ITS controller for MSI interrupts. + */ +#define GITS_CTLR 0x0000 +#define GITS_CTLR_ENABLED (1UL << 0) +#define GITS_TYPER 0x0008 +#define GITS_TYPER_CIL (1ULL << 36) +#define GITS_TYPER_HCC(x) (((x) >> 24) & 0xff) +#define GITS_TYPER_PTA (1ULL << 19) +#define GITS_TYPER_ITE_SZ(x) (((x) >> 4) & 0xf) +#define GITS_TYPER_PHYS (1ULL << 0) +#define GITS_CBASER 0x0080 +#define GITS_CBASER_VALID (1ULL << 63) +#define GITS_CBASER_IC_NORM_NC (1ULL << 59) +#define GITS_CBASER_MASK 0x1ffffffffff000ULL +#define GITS_CWRITER 0x0088 +#define GITS_CREADR 0x0090 +#define GITS_BASER(i) (0x0100 + ((i) * 8)) +#define GITS_BASER_VALID (1ULL << 63) +#define GITS_BASER_INDIRECT (1ULL << 62) +#define GITS_BASER_IC_NORM_NC (1ULL << 59) +#define GITS_BASER_TYPE_MASK (7ULL << 56) +#define GITS_BASER_TYPE_DEVICE (1ULL << 56) +#define GITS_BASER_MASK 0x7ffffffff000ULL +#define GITS_TRANSLATER 0x10040 + +struct gits_cmd { + uint8_t cmd; + uint32_t deviceid; + uint32_t eventid; + uint32_t intid; + uint64_t dw2; + uint64_t dw3; +}; + +#define GITS_CMD_VALID (1ULL << 63) + +/* ITS commands */ +#define SYNC 0x05 +#define MAPD 0x08 +#define MAPC 0x09 +#define MAPTI 0x0a + +#define GITS_CMDQ_SIZE (64 * 1024) +#define GITS_CMDQ_NENTRIES (GITS_CMDQ_SIZE / sizeof(struct gits_cmd)) + +#define GITS_DTT_SIZE (64 * 1024) + +struct agintc_msi_device { + LIST_ENTRY(agintc_msi_device) md_list; + + uint32_t md_deviceid; + uint32_t md_eventid; + struct agintc_dmamem *md_itt; +}; + +int agintc_msi_match(struct device *, void *, void *); +void agintc_msi_attach(struct device *, struct device *, void *); +void *agintc_intr_establish_msi(void *, uint64_t *, uint64_t *, + int , int (*)(void *), void *, char *); +void agintc_intr_disestablish_msi(void *); + +struct agintc_msi_softc { + struct device sc_dev; + bus_space_tag_t sc_iot; + bus_space_handle_t sc_ioh; + bus_addr_t sc_addr; + bus_dma_tag_t sc_dmat; + + int sc_nlpi; + void **sc_lpi; + + struct agintc_dmamem *sc_cmdq; + uint16_t sc_cmdidx; + struct agintc_dmamem *sc_dtt; + uint8_t sc_ite_sz; + + LIST_HEAD(, agintc_msi_device) sc_msi_devices; + + struct interrupt_controller sc_ic; +}; + +struct cfattach agintcmsi_ca = { + sizeof (struct agintc_msi_softc), agintc_msi_match, agintc_msi_attach +}; + +struct cfdriver agintcmsi_cd = { + NULL, "agintcmsi", DV_DULL +}; + +void agintc_msi_send_cmd(struct agintc_msi_softc *, struct gits_cmd *); +void agintc_msi_wait_cmd(struct agintc_msi_softc *); + +int +agintc_msi_match(struct device *parent, void *cfdata, void *aux) +{ + struct fdt_attach_args *faa = aux; + + return OF_is_compatible(faa->fa_node, "arm,gic-v3-its"); +} + +void +agintc_msi_attach(struct device *parent, struct device *self, void *aux) +{ + struct agintc_msi_softc *sc = (struct agintc_msi_softc *)self; + struct fdt_attach_args *faa = aux; + struct gits_cmd cmd; + uint64_t typer; + int i; + + if (faa->fa_nreg < 1) { + printf(": no registers\n"); + return; + } + + sc->sc_iot = faa->fa_iot; + if (bus_space_map(sc->sc_iot, faa->fa_reg[0].addr, + faa->fa_reg[0].size, 0, &sc->sc_ioh)) { + printf(": can't map registers\n"); + return; + } + sc->sc_addr = faa->fa_reg[0].addr; + sc->sc_dmat = faa->fa_dmat; + + typer = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_TYPER); + if ((typer & GITS_TYPER_PHYS) == 0 || typer & GITS_TYPER_PTA || + GITS_TYPER_HCC(typer) == 0 || typer & GITS_TYPER_CIL) { + printf(": unsupported type 0x%016llx\n", typer); + goto unmap; + } + sc->sc_ite_sz = GITS_TYPER_ITE_SZ(typer) + 1; + + sc->sc_nlpi = agintc_sc->sc_nlpi; + sc->sc_lpi = mallocarray(sc->sc_nlpi, sizeof(void *), M_DEVBUF, + M_WAITOK|M_ZERO); + + /* Set up command queue. */ + sc->sc_cmdq = agintc_dmamem_alloc(sc->sc_dmat, + GITS_CMDQ_SIZE, GITS_CMDQ_SIZE); + if (sc->sc_cmdq == NULL) { + printf(": can't alloc command queue\n"); + goto unmap; + } + bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_CBASER, + AGINTC_DMA_DVA(sc->sc_cmdq) | GITS_CBASER_IC_NORM_NC | + (GITS_CMDQ_SIZE / PAGE_SIZE) - 1 | GITS_CBASER_VALID); + + /* Set up device translation table. */ + for (i = 0; i < 8; i++) { + uint64_t baser; + + baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); + if ((baser & GITS_BASER_TYPE_MASK) != GITS_BASER_TYPE_DEVICE) + continue; + + sc->sc_dtt = agintc_dmamem_alloc(sc->sc_dmat, + GITS_DTT_SIZE, GITS_DTT_SIZE); + if (sc->sc_dtt == NULL) { + printf(": can't alloc translation table\n"); + goto unmap; + } + bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(0), + AGINTC_DMA_DVA(sc->sc_dtt) | GITS_BASER_IC_NORM_NC | + (GITS_DTT_SIZE / PAGE_SIZE) - 1 | GITS_BASER_VALID); + } + + /* Enable ITS. */ + bus_space_write_4(sc->sc_iot, sc->sc_ioh, GITS_CTLR, + GITS_CTLR_ENABLED); + + LIST_INIT(&sc->sc_msi_devices); + + /* Map collection 0 to redistributor 0. */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = MAPC; + cmd.dw2 = GITS_CMD_VALID; + agintc_msi_send_cmd(sc, &cmd); + agintc_msi_wait_cmd(sc); + + printf("\n"); + + sc->sc_ic.ic_node = faa->fa_node; + sc->sc_ic.ic_cookie = sc; + sc->sc_ic.ic_establish_msi = agintc_intr_establish_msi; + sc->sc_ic.ic_disestablish = agintc_intr_disestablish_msi; + arm_intr_register_fdt(&sc->sc_ic); + return; + +unmap: + if (sc->sc_dtt) + agintc_dmamem_free(sc->sc_dmat, sc->sc_dtt); + if (sc->sc_cmdq) + agintc_dmamem_free(sc->sc_dmat, sc->sc_cmdq); + + if (sc->sc_lpi) + free(sc->sc_lpi, M_DEVBUF, sc->sc_nlpi * sizeof(void *)); + + bus_space_unmap(sc->sc_iot, sc->sc_ioh, faa->fa_reg[0].size); +} + +void +agintc_msi_send_cmd(struct agintc_msi_softc *sc, struct gits_cmd *cmd) +{ + struct gits_cmd *queue = AGINTC_DMA_KVA(sc->sc_cmdq); + + memcpy(&queue[sc->sc_cmdidx++], cmd, sizeof(*cmd)); + __asm volatile("dsb sy"); + + sc->sc_cmdidx %= GITS_CMDQ_NENTRIES; + bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_CWRITER, + sc->sc_cmdidx * sizeof(*cmd)); +} + +void +agintc_msi_wait_cmd(struct agintc_msi_softc *sc) +{ + uint64_t creadr; + int timo; + + for (timo = 1000; timo > 0; timo--) { + creadr = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_CREADR); + if (creadr == sc->sc_cmdidx * sizeof(struct gits_cmd)) + break; + delay(1); + } + if (timo == 0) + printf("%s: command queue timeout\n", sc->sc_dev.dv_xname); +} + +struct agintc_msi_device * +agintc_msi_create_device(struct agintc_msi_softc *sc, uint32_t deviceid) +{ + struct agintc_msi_device *md; + struct gits_cmd cmd; + + md = malloc(sizeof(*md), M_DEVBUF, M_ZERO | M_WAITOK); + md->md_deviceid = deviceid; + md->md_itt = agintc_dmamem_alloc(sc->sc_dmat, + 32 * sc->sc_ite_sz, PAGE_SIZE); + LIST_INSERT_HEAD(&sc->sc_msi_devices, md, md_list); + + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = MAPD; + cmd.deviceid = deviceid; + cmd.eventid = 4; /* size */ + cmd.dw2 = AGINTC_DMA_DVA(md->md_itt) | GITS_CMD_VALID; + agintc_msi_send_cmd(sc, &cmd); + agintc_msi_wait_cmd(sc); + + return md; +} + +struct agintc_msi_device * +agintc_msi_find_device(struct agintc_msi_softc *sc, uint32_t deviceid) +{ + struct agintc_msi_device *md; + + LIST_FOREACH(md, &sc->sc_msi_devices, md_list) { + if (md->md_deviceid == deviceid) + return md; + } + + return agintc_msi_create_device(sc, deviceid); +} + +void * +agintc_intr_establish_msi(void *self, uint64_t *addr, uint64_t *data, + int level, int (*func)(void *), void *arg, char *name) +{ + struct agintc_msi_softc *sc = (struct agintc_msi_softc *)self; + struct agintc_msi_device *md; + struct gits_cmd cmd; + uint32_t deviceid = *data; + uint32_t eventid; + void *cookie; + int i; + + md = agintc_msi_find_device(sc, deviceid); + if (md == NULL) + return NULL; + + eventid = md->md_eventid++; + if (eventid >= 32) + return NULL; + + for (i = 0; i < sc->sc_nlpi; i++) { + if (sc->sc_lpi[i] != NULL) + continue; + + cookie = agintc_intr_establish(LPI_BASE + i, + level, func, arg, name); + if (cookie == NULL) + return NULL; + + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = MAPTI; + cmd.deviceid = deviceid; + cmd.eventid = eventid; + cmd.intid = LPI_BASE + i; + cmd.dw2 = GITS_CMD_VALID; + agintc_msi_send_cmd(sc, &cmd); + + memset(&cmd, 0, sizeof(cmd)); + cmd.cmd = SYNC; + cmd.dw2 = 0; + agintc_msi_send_cmd(sc, &cmd); + agintc_msi_wait_cmd(sc); + + *addr = sc->sc_addr + GITS_TRANSLATER; + *data = eventid; + sc->sc_lpi[i] = cookie; + return &sc->sc_lpi[i]; + } + + return NULL; +} + +void +agintc_intr_disestablish_msi(void *cookie) +{ + agintc_intr_disestablish(*(void **)cookie); + *(void **)cookie = NULL; +} + +struct agintc_dmamem * +agintc_dmamem_alloc(bus_dma_tag_t dmat, bus_size_t size, bus_size_t align) +{ + struct agintc_dmamem *adm; + int nsegs; + + adm = malloc(sizeof(*adm), M_DEVBUF, M_WAITOK | M_ZERO); + adm->adm_size = size; + + if (bus_dmamap_create(dmat, size, 1, size, 0, + BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0) + goto admfree; + + if (bus_dmamem_alloc(dmat, size, align, 0, &adm->adm_seg, 1, + &nsegs, BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) + goto destroy; + + if (bus_dmamem_map(dmat, &adm->adm_seg, nsegs, size, + &adm->adm_kva, BUS_DMA_WAITOK | BUS_DMA_NOCACHE) != 0) + goto free; + + if (bus_dmamap_load_raw(dmat, adm->adm_map, &adm->adm_seg, + nsegs, size, BUS_DMA_WAITOK) != 0) + goto unmap; + + return adm; + +unmap: + bus_dmamem_unmap(dmat, adm->adm_kva, size); +free: + bus_dmamem_free(dmat, &adm->adm_seg, 1); +destroy: + bus_dmamap_destroy(dmat, adm->adm_map); +admfree: + free(adm, M_DEVBUF, sizeof(*adm)); + + return NULL; +} + +void +agintc_dmamem_free(bus_dma_tag_t dmat, struct agintc_dmamem *adm) +{ + bus_dmamem_unmap(dmat, adm->adm_kva, adm->adm_size); + bus_dmamem_free(dmat, &adm->adm_seg, 1); + bus_dmamap_destroy(dmat, adm->adm_map); + free(adm, M_DEVBUF, sizeof(*adm)); +} diff --git a/sys/dev/fdt/rkpcie.c b/sys/dev/fdt/rkpcie.c index c7ff5fcd43c..615ac3aba21 100644 --- a/sys/dev/fdt/rkpcie.c +++ b/sys/dev/fdt/rkpcie.c @@ -1,4 +1,4 @@ -/* $OpenBSD: rkpcie.c,v 1.3 2018/01/13 18:08:20 kettenis Exp $ */ +/* $OpenBSD: rkpcie.c,v 1.4 2018/07/30 10:56:00 kettenis Exp $ */ /* * Copyright (c) 2018 Mark Kettenis * @@ -306,6 +306,7 @@ rkpcie_attach(struct device *parent, struct device *self, void *aux) pba.pba_ioex = sc->sc_ioex; pba.pba_domain = pci_ndomains++; pba.pba_bus = sc->sc_bus; + pba.pba_flags |= PCI_FLAGS_MSI_ENABLED; config_found(self, &pba, NULL); } @@ -478,6 +479,7 @@ struct rkpcie_intr_handle { pci_chipset_tag_t ih_pc; pcitag_t ih_tag; int ih_intrpin; + int ih_msi; }; int @@ -496,6 +498,7 @@ rkpcie_intr_map(struct pci_attach_args *pa, pci_intr_handle_t *ihp) ih->ih_pc = pa->pa_pc; ih->ih_tag = pa->pa_intrtag; ih->ih_intrpin = pa->pa_intrpin; + ih->ih_msi = 0; *ihp = (pci_intr_handle_t)ih; return 0; @@ -506,11 +509,20 @@ rkpcie_intr_map_msi(struct pci_attach_args *pa, pci_intr_handle_t *ihp) { pci_chipset_tag_t pc = pa->pa_pc; pcitag_t tag = pa->pa_tag; + struct rkpcie_intr_handle *ih; - if (pci_get_capability(pc, tag, PCI_CAP_MSI, NULL, NULL) == 0) + if ((pa->pa_flags & PCI_FLAGS_MSI_ENABLED) == 0 || + pci_get_capability(pc, tag, PCI_CAP_MSI, NULL, NULL) == 0) return -1; - return -1; + ih = malloc(sizeof(struct rkpcie_intr_handle), M_DEVBUF, M_WAITOK); + ih->ih_pc = pa->pa_pc; + ih->ih_tag = pa->pa_tag; + ih->ih_intrpin = pa->pa_intrpin; + ih->ih_msi = 1; + *ihp = (pci_intr_handle_t)ih; + + return 0; } int @@ -521,24 +533,69 @@ rkpcie_intr_map_msix(struct pci_attach_args *pa, int vec, } const char * -rkpcie_intr_string(void *v, pci_intr_handle_t ih) +rkpcie_intr_string(void *v, pci_intr_handle_t ihp) { + struct rkpcie_intr_handle *ih = (struct rkpcie_intr_handle *)ihp; + + if (ih->ih_msi) + return "msi"; + return "intx"; } void * -rkpcie_intr_establish(void *v, pci_intr_handle_t ih, int level, +rkpcie_intr_establish(void *v, pci_intr_handle_t ihp, int level, int (*func)(void *), void *arg, char *name) { struct rkpcie_softc *sc = v; + struct rkpcie_intr_handle *ih = (struct rkpcie_intr_handle *)ihp; + void *cookie; + + if (ih->ih_msi) { + uint64_t addr, data; + pcireg_t reg; + int off; + + /* Assume hardware passes Requester ID as sideband data. */ + data = pci_requester_id(ih->ih_pc, ih->ih_tag); + cookie = arm_intr_establish_fdt_msi(sc->sc_node, &addr, + &data, level, func, arg, name); + if (cookie == NULL) + return NULL; + + /* TODO: translate address to the PCI device's view */ + + if (pci_get_capability(ih->ih_pc, ih->ih_tag, PCI_CAP_MSI, + &off, ®) == 0) + panic("%s: no msi capability", __func__); + + if (reg & PCI_MSI_MC_C64) { + pci_conf_write(ih->ih_pc, ih->ih_tag, + off + PCI_MSI_MA, addr); + pci_conf_write(ih->ih_pc, ih->ih_tag, + off + PCI_MSI_MAU32, addr >> 32); + pci_conf_write(ih->ih_pc, ih->ih_tag, + off + PCI_MSI_MD64, data); + } else { + pci_conf_write(ih->ih_pc, ih->ih_tag, + off + PCI_MSI_MA, addr); + pci_conf_write(ih->ih_pc, ih->ih_tag, + off + PCI_MSI_MD32, data); + } + pci_conf_write(ih->ih_pc, ih->ih_tag, + off, reg | PCI_MSI_MC_MSIE); + } else { + /* Unmask legacy interrupts. */ + HWRITE4(sc, PCIE_CLIENT_INT_MASK, + PCIE_CLIENT_INTA_UNMASK | PCIE_CLIENT_INTB_UNMASK | + PCIE_CLIENT_INTC_UNMASK | PCIE_CLIENT_INTD_UNMASK); + + cookie = arm_intr_establish_fdt_idx(sc->sc_node, 1, level, + func, arg, name); + } - /* Unmask legacy interrupts. */ - HWRITE4(sc, PCIE_CLIENT_INT_MASK, - PCIE_CLIENT_INTA_UNMASK | PCIE_CLIENT_INTB_UNMASK | - PCIE_CLIENT_INTC_UNMASK | PCIE_CLIENT_INTD_UNMASK); - - return arm_intr_establish_fdt_idx(sc->sc_node, 1, level, - func, arg, name); + free(ih, M_DEVBUF, sizeof(struct rkpcie_intr_handle)); + return cookie; } void