From fb432fadab0adf2614ada0a52d3e164cf3545a90 Mon Sep 17 00:00:00 2001 From: kettenis Date: Fri, 19 Jan 2024 18:38:16 +0000 Subject: [PATCH] Implement Multiple Message MSI support on amd64. This is experimental code to assist qwx(4) development. We may remove this code again at some point in the future. Multiple Message MSI has some serious design flaws, especially when combined with the APIC interrupt controller architecture. It was superseded by MSI-X. Unfortunately qwx(4) does not implement MSI-X. ok stsp@, deraadt@ --- sys/arch/amd64/amd64/i8259.c | 3 +- sys/arch/amd64/amd64/intr.c | 13 ++- sys/arch/amd64/amd64/machdep.c | 25 ++++- sys/arch/amd64/include/pci_machdep.h | 5 +- sys/arch/amd64/include/pic.h | 3 +- sys/arch/amd64/include/segments.h | 3 +- sys/arch/amd64/pci/pci_machdep.c | 159 ++++++++++++++++++++++----- sys/dev/pci/pcireg.h | 8 +- 8 files changed, 181 insertions(+), 38 deletions(-) diff --git a/sys/arch/amd64/amd64/i8259.c b/sys/arch/amd64/amd64/i8259.c index 875d22053ef..d6fa0e3d7f3 100644 --- a/sys/arch/amd64/amd64/i8259.c +++ b/sys/arch/amd64/amd64/i8259.c @@ -1,4 +1,4 @@ -/* $OpenBSD: i8259.c,v 1.11 2018/07/27 21:11:31 kettenis Exp $ */ +/* $OpenBSD: i8259.c,v 1.12 2024/01/19 18:38:16 kettenis Exp $ */ /* $NetBSD: i8259.c,v 1.2 2003/03/02 18:27:15 fvdl Exp $ */ /* @@ -100,6 +100,7 @@ struct pic i8259_pic = { i8259_hwunmask, i8259_setup, i8259_setup, + NULL, i8259_stubs, i8259_stubs, }; diff --git a/sys/arch/amd64/amd64/intr.c b/sys/arch/amd64/amd64/intr.c index a3331810a6a..64396b3899a 100644 --- a/sys/arch/amd64/amd64/intr.c +++ b/sys/arch/amd64/amd64/intr.c @@ -1,4 +1,4 @@ -/* $OpenBSD: intr.c,v 1.55 2020/12/28 14:23:30 mpi Exp $ */ +/* $OpenBSD: intr.c,v 1.56 2024/01/19 18:38:16 kettenis Exp $ */ /* $NetBSD: intr.c,v 1.3 2003/03/03 22:16:20 fvdl Exp $ */ /* @@ -310,9 +310,16 @@ other: } return EBUSY; found: - idtvec = idt_vec_alloc(APIC_LEVEL(level), IDT_INTR_HIGH); + if (pic->pic_allocidtvec) { + idtvec = pic->pic_allocidtvec(pic, pin, + APIC_LEVEL(level), IDT_INTR_HIGH); + } else { + idtvec = idt_vec_alloc(APIC_LEVEL(level), + IDT_INTR_HIGH); + } if (idtvec == 0) { - free(ci->ci_isources[slot], M_DEVBUF, sizeof (struct intrsource)); + free(ci->ci_isources[slot], M_DEVBUF, + sizeof (struct intrsource)); ci->ci_isources[slot] = NULL; return EBUSY; } diff --git a/sys/arch/amd64/amd64/machdep.c b/sys/arch/amd64/amd64/machdep.c index 606630a9f0a..ec472ca48bd 100644 --- a/sys/arch/amd64/amd64/machdep.c +++ b/sys/arch/amd64/amd64/machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: machdep.c,v 1.288 2023/09/08 20:47:22 kn Exp $ */ +/* $OpenBSD: machdep.c,v 1.289 2024/01/19 18:38:16 kettenis Exp $ */ /* $NetBSD: machdep.c,v 1.3 2003/05/07 22:58:18 fvdl Exp $ */ /*- @@ -1918,6 +1918,29 @@ idt_vec_alloc(int low, int high) return 0; } +int +idt_vec_alloc_range(int low, int high, int num) +{ + int i, vec; + + KASSERT(powerof2(num)); + low = (low + num - 1) & ~(num - 1); + high = ((high + 1) & ~(num - 1)) - 1; + + for (vec = low; vec <= high; vec += num) { + for (i = 0; i < num; i++) { + if (idt_allocmap[vec + i] != 0) + break; + } + if (i == num) { + for (i = 0; i < num; i++) + idt_allocmap[vec + i] = 1; + return vec; + } + } + return 0; +} + void idt_vec_set(int vec, void (*function)(void)) { diff --git a/sys/arch/amd64/include/pci_machdep.h b/sys/arch/amd64/include/pci_machdep.h index 6e5928ec8fd..9abd854f985 100644 --- a/sys/arch/amd64/include/pci_machdep.h +++ b/sys/arch/amd64/include/pci_machdep.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pci_machdep.h,v 1.30 2020/10/27 02:39:07 jordan Exp $ */ +/* $OpenBSD: pci_machdep.h,v 1.31 2024/01/19 18:38:16 kettenis Exp $ */ /* $NetBSD: pci_machdep.h,v 1.1 2003/02/26 21:26:11 fvdl Exp $ */ /* @@ -79,8 +79,11 @@ int pci_conf_size(pci_chipset_tag_t, pcitag_t); pcireg_t pci_conf_read(pci_chipset_tag_t, pcitag_t, int); void pci_conf_write(pci_chipset_tag_t, pcitag_t, int, pcireg_t); +int pci_intr_enable_msivec(struct pci_attach_args *, int); int pci_intr_map_msi(struct pci_attach_args *, pci_intr_handle_t *); +int pci_intr_map_msivec(struct pci_attach_args *, + int, pci_intr_handle_t *); int pci_intr_map_msix(struct pci_attach_args *, int, pci_intr_handle_t *); int pci_intr_map(struct pci_attach_args *, pci_intr_handle_t *); diff --git a/sys/arch/amd64/include/pic.h b/sys/arch/amd64/include/pic.h index 59846fbeae6..c5b29937dec 100644 --- a/sys/arch/amd64/include/pic.h +++ b/sys/arch/amd64/include/pic.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pic.h,v 1.7 2014/12/16 21:20:23 tedu Exp $ */ +/* $OpenBSD: pic.h,v 1.8 2024/01/19 18:38:16 kettenis Exp $ */ /* $NetBSD: pic.h,v 1.1 2003/02/26 21:26:11 fvdl Exp $ */ #ifndef _X86_PIC_H @@ -22,6 +22,7 @@ struct pic { void (*pic_hwunmask)(struct pic *, int); void (*pic_addroute)(struct pic *, struct cpu_info *, int, int, int); void (*pic_delroute)(struct pic *, struct cpu_info *, int, int, int); + int (*pic_allocidtvec)(struct pic *, int, int, int); struct intrstub *pic_level_stubs; struct intrstub *pic_edge_stubs; }; diff --git a/sys/arch/amd64/include/segments.h b/sys/arch/amd64/include/segments.h index 6ce184e88d1..87de2c08240 100644 --- a/sys/arch/amd64/include/segments.h +++ b/sys/arch/amd64/include/segments.h @@ -1,4 +1,4 @@ -/* $OpenBSD: segments.h,v 1.15 2018/03/29 01:21:02 guenther Exp $ */ +/* $OpenBSD: segments.h,v 1.16 2024/01/19 18:38:16 kettenis Exp $ */ /* $NetBSD: segments.h,v 1.1 2003/04/26 18:39:47 fvdl Exp $ */ /*- @@ -160,6 +160,7 @@ void set_sys_segment(struct sys_segment_descriptor *, void *, size_t, void set_mem_segment(struct mem_segment_descriptor *, void *, size_t, int, int, int, int, int); int idt_vec_alloc(int, int); +int idt_vec_alloc_range(int, int, int); void idt_vec_set(int, void (*)(void)); void idt_vec_free(int); void cpu_init_idt(void); diff --git a/sys/arch/amd64/pci/pci_machdep.c b/sys/arch/amd64/pci/pci_machdep.c index 72456c32829..41c65293c4d 100644 --- a/sys/arch/amd64/pci/pci_machdep.c +++ b/sys/arch/amd64/pci/pci_machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: pci_machdep.c,v 1.77 2021/03/11 11:16:55 jsg Exp $ */ +/* $OpenBSD: pci_machdep.c,v 1.78 2024/01/19 18:38:16 kettenis Exp $ */ /* $NetBSD: pci_machdep.c,v 1.3 2003/05/07 21:33:58 fvdl Exp $ */ /*- @@ -315,10 +315,21 @@ pci_msix_table_unmap(pci_chipset_tag_t pc, pcitag_t tag, _bus_space_unmap(memt, memh, tblsz * 16, NULL); } +/* + * We pack the MSI vector number into the lower 8 bits of the PCI tag + * and use that as the MSI/MSI-X "PIC" pin number. This allows us to + * address 256 MSI vectors which ought to be enough for anybody. + */ +#define PCI_MSI_VEC_MASK 0xff +#define PCI_MSI_VEC(pin) ((pin) & PCI_MSI_VEC_MASK) +#define PCI_MSI_TAG(pin) ((pin) & ~PCI_MSI_VEC_MASK) +#define PCI_MSI_PIN(tag, vec) ((tag) | (vec)) + void msi_hwmask(struct pic *, int); void msi_hwunmask(struct pic *, int); void msi_addroute(struct pic *, struct cpu_info *, int, int, int); void msi_delroute(struct pic *, struct cpu_info *, int, int, int); +int msi_allocidtvec(struct pic *, int, int, int); struct pic msi_pic = { {0, {NULL}, NULL, 0, "msi", NULL, 0, 0}, @@ -330,6 +341,7 @@ struct pic msi_pic = { msi_hwunmask, msi_addroute, msi_delroute, + msi_allocidtvec, NULL, ioapic_edge_stubs }; @@ -345,57 +357,157 @@ msi_hwunmask(struct pic *pic, int pin) } void -msi_addroute(struct pic *pic, struct cpu_info *ci, int pin, int vec, int type) +msi_addroute(struct pic *pic, struct cpu_info *ci, int pin, int idtvec, + int type) { pci_chipset_tag_t pc = NULL; /* XXX */ - pcitag_t tag = pin; + pcitag_t tag = PCI_MSI_TAG(pin); + int vec = PCI_MSI_VEC(pin); pcireg_t reg, addr; int off; if (pci_get_capability(pc, tag, PCI_CAP_MSI, &off, ®) == 0) panic("%s: no msi capability", __func__); + if (vec != 0) + return; + addr = 0xfee00000UL | (ci->ci_apicid << 12); if (reg & PCI_MSI_MC_C64) { pci_conf_write(pc, tag, off + PCI_MSI_MA, addr); pci_conf_write(pc, tag, off + PCI_MSI_MAU32, 0); - pci_conf_write(pc, tag, off + PCI_MSI_MD64, vec); + pci_conf_write(pc, tag, off + PCI_MSI_MD64, idtvec); } else { pci_conf_write(pc, tag, off + PCI_MSI_MA, addr); - pci_conf_write(pc, tag, off + PCI_MSI_MD32, vec); + pci_conf_write(pc, tag, off + PCI_MSI_MD32, idtvec); } pci_conf_write(pc, tag, off, reg | PCI_MSI_MC_MSIE); } void -msi_delroute(struct pic *pic, struct cpu_info *ci, int pin, int vec, int type) +msi_delroute(struct pic *pic, struct cpu_info *ci, int pin, int idtvec, + int type) { pci_chipset_tag_t pc = NULL; /* XXX */ - pcitag_t tag = pin; + pcitag_t tag = PCI_MSI_TAG(pin); + int vec = PCI_MSI_VEC(pin); pcireg_t reg; int off; + if (vec != 0) + return; + if (pci_get_capability(pc, tag, PCI_CAP_MSI, &off, ®)) pci_conf_write(pc, tag, off, reg & ~PCI_MSI_MC_MSIE); } +int +msi_allocidtvec(struct pic *pic, int pin, int low, int high) +{ + pci_chipset_tag_t pc = NULL; /* XXX */ + pcitag_t tag = PCI_MSI_TAG(pin); + int vec = PCI_MSI_VEC(pin); + int idtvec, mme, off; + pcireg_t reg; + + if (pci_get_capability(pc, tag, PCI_CAP_MSI, &off, ®) == 0) + panic("%s: no msi capability", __func__); + + reg = pci_conf_read(pc, tag, off); + mme = ((reg & PCI_MSI_MC_MME_MASK) >> PCI_MSI_MC_MME_SHIFT); + if (vec >= (1 << mme)) + return 0; + + if (vec == 0) { + idtvec = idt_vec_alloc_range(low, high, (1 << mme)); + if (reg & PCI_MSI_MC_C64) + pci_conf_write(pc, tag, off + PCI_MSI_MD64, idtvec); + else + pci_conf_write(pc, tag, off + PCI_MSI_MD32, idtvec); + } else { + if (reg & PCI_MSI_MC_C64) + reg = pci_conf_read(pc, tag, off + PCI_MSI_MD64); + else + reg = pci_conf_read(pc, tag, off + PCI_MSI_MD32); + KASSERT(reg > 0); + idtvec = reg + vec; + } + + return idtvec; +} + +int +pci_intr_enable_msivec(struct pci_attach_args *pa, int num_vec) +{ + pci_chipset_tag_t pc = pa->pa_pc; + pcitag_t tag = pa->pa_tag; + pcireg_t reg; + int mmc, mme, off; + + if ((pa->pa_flags & PCI_FLAGS_MSI_ENABLED) == 0 || mp_busses == NULL || + pci_get_capability(pc, tag, PCI_CAP_MSI, &off, ®) == 0) + return 1; + + mmc = ((reg & PCI_MSI_MC_MMC_MASK) >> PCI_MSI_MC_MMC_SHIFT); + if (num_vec > (1 << mmc)) + return 1; + + mme = ((reg & PCI_MSI_MC_MME_MASK) >> PCI_MSI_MC_MME_SHIFT); + while ((1 << mme) < num_vec) + mme++; + reg &= ~PCI_MSI_MC_MME_MASK; + reg |= (mme << PCI_MSI_MC_MME_SHIFT); + pci_conf_write(pc, tag, off, reg); + + return 0; +} + int pci_intr_map_msi(struct pci_attach_args *pa, pci_intr_handle_t *ihp) { pci_chipset_tag_t pc = pa->pa_pc; pcitag_t tag = pa->pa_tag; + pcireg_t reg; + int off; if ((pa->pa_flags & PCI_FLAGS_MSI_ENABLED) == 0 || mp_busses == NULL || - pci_get_capability(pc, tag, PCI_CAP_MSI, NULL, NULL) == 0) + pci_get_capability(pc, tag, PCI_CAP_MSI, &off, ®) == 0) return 1; + /* Make sure we only enable one MSI vector. */ + reg &= ~PCI_MSI_MC_MME_MASK; + pci_conf_write(pc, tag, off, reg); + ihp->tag = tag; ihp->line = APIC_INT_VIA_MSG; ihp->pin = 0; return 0; } +int +pci_intr_map_msivec(struct pci_attach_args *pa, int vec, + pci_intr_handle_t *ihp) +{ + pci_chipset_tag_t pc = pa->pa_pc; + pcitag_t tag = pa->pa_tag; + pcireg_t reg; + int mme, off; + + if ((pa->pa_flags & PCI_FLAGS_MSI_ENABLED) == 0 || mp_busses == NULL || + pci_get_capability(pc, tag, PCI_CAP_MSI, &off, ®) == 0) + return 1; + + mme = ((reg & PCI_MSI_MC_MME_MASK) >> PCI_MSI_MC_MME_SHIFT); + if (vec > (1 << mme)) + return 0; + + ihp->tag = PCI_MSI_PIN(tag, vec); + ihp->line = APIC_INT_VIA_MSG; + ihp->pin = 0; + return 0; +} + void msix_hwmask(struct pic *, int); void msix_hwunmask(struct pic *, int); void msix_addroute(struct pic *, struct cpu_info *, int, int, int); @@ -412,19 +524,10 @@ struct pic msix_pic = { msix_addroute, msix_delroute, NULL, + NULL, ioapic_edge_stubs }; -/* - * We pack the MSI-X vector number into the lower 8 bits of the PCI - * tag and use that as the MSI-X "PIC" pin number. This allows us to - * address 256 MSI-X vectors which ought to be enough for anybody. - */ -#define PCI_MSIX_VEC_MASK 0xff -#define PCI_MSIX_VEC(pin) ((pin) & PCI_MSIX_VEC_MASK) -#define PCI_MSIX_TAG(pin) ((pin) & ~PCI_MSIX_VEC_MASK) -#define PCI_MSIX_PIN(tag, vec) ((tag) | (vec)) - void msix_hwmask(struct pic *pic, int pin) { @@ -436,13 +539,14 @@ msix_hwunmask(struct pic *pic, int pin) } void -msix_addroute(struct pic *pic, struct cpu_info *ci, int pin, int vec, int type) +msix_addroute(struct pic *pic, struct cpu_info *ci, int pin, int idtvec, + int type) { pci_chipset_tag_t pc = NULL; /* XXX */ bus_space_tag_t memt = X86_BUS_SPACE_MEM; /* XXX */ bus_space_handle_t memh; - pcitag_t tag = PCI_MSIX_TAG(pin); - int entry = PCI_MSIX_VEC(pin); + pcitag_t tag = PCI_MSI_TAG(pin); + int entry = PCI_MSI_VEC(pin); pcireg_t reg, addr; uint32_t ctrl; int off; @@ -459,7 +563,7 @@ msix_addroute(struct pic *pic, struct cpu_info *ci, int pin, int vec, int type) bus_space_write_4(memt, memh, PCI_MSIX_MA(entry), addr); bus_space_write_4(memt, memh, PCI_MSIX_MAU32(entry), 0); - bus_space_write_4(memt, memh, PCI_MSIX_MD(entry), vec); + bus_space_write_4(memt, memh, PCI_MSIX_MD(entry), idtvec); bus_space_barrier(memt, memh, PCI_MSIX_MA(entry), 16, BUS_SPACE_BARRIER_WRITE); ctrl = bus_space_read_4(memt, memh, PCI_MSIX_VC(entry)); @@ -472,13 +576,14 @@ msix_addroute(struct pic *pic, struct cpu_info *ci, int pin, int vec, int type) } void -msix_delroute(struct pic *pic, struct cpu_info *ci, int pin, int vec, int type) +msix_delroute(struct pic *pic, struct cpu_info *ci, int pin, int idtvec, + int type) { pci_chipset_tag_t pc = NULL; /* XXX */ bus_space_tag_t memt = X86_BUS_SPACE_MEM; /* XXX */ bus_space_handle_t memh; - pcitag_t tag = PCI_MSIX_TAG(pin); - int entry = PCI_MSIX_VEC(pin); + pcitag_t tag = PCI_MSI_TAG(pin); + int entry = PCI_MSI_VEC(pin); pcireg_t reg; uint32_t ctrl; @@ -504,7 +609,7 @@ pci_intr_map_msix(struct pci_attach_args *pa, int vec, pci_intr_handle_t *ihp) pcitag_t tag = pa->pa_tag; pcireg_t reg; - KASSERT(PCI_MSIX_VEC(vec) == vec); + KASSERT(PCI_MSI_VEC(vec) == vec); if ((pa->pa_flags & PCI_FLAGS_MSI_ENABLED) == 0 || mp_busses == NULL || pci_get_capability(pc, tag, PCI_CAP_MSIX, NULL, ®) == 0) @@ -513,7 +618,7 @@ pci_intr_map_msix(struct pci_attach_args *pa, int vec, pci_intr_handle_t *ihp) if (vec > PCI_MSIX_MC_TBLSZ(reg)) return 1; - ihp->tag = PCI_MSIX_PIN(tag, vec); + ihp->tag = PCI_MSI_PIN(tag, vec); ihp->line = APIC_INT_VIA_MSGX; ihp->pin = 0; return 0; diff --git a/sys/dev/pci/pcireg.h b/sys/dev/pci/pcireg.h index 53124eccecb..b839bee9112 100644 --- a/sys/dev/pci/pcireg.h +++ b/sys/dev/pci/pcireg.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pcireg.h,v 1.61 2022/06/17 10:08:36 kettenis Exp $ */ +/* $OpenBSD: pcireg.h,v 1.62 2024/01/19 18:38:16 kettenis Exp $ */ /* $NetBSD: pcireg.h,v 1.26 2000/05/10 16:58:42 thorpej Exp $ */ /* @@ -516,8 +516,10 @@ typedef u_int8_t pci_revision_t; */ #define PCI_MSI_MC 0x00 #define PCI_MSI_MC_C64 0x00800000 -#define PCI_MSI_MC_MME 0x00700000 -#define PCI_MSI_MC_MMC 0x000e0000 +#define PCI_MSI_MC_MME_MASK 0x00700000 +#define PCI_MSI_MC_MME_SHIFT 20 +#define PCI_MSI_MC_MMC_MASK 0x000e0000 +#define PCI_MSI_MC_MMC_SHIFT 17 #define PCI_MSI_MC_MSIE 0x00010000 #define PCI_MSI_MA 0x04 #define PCI_MSI_MAU32 0x08 -- 2.20.1