From adc7c282072bd0a8067bf9fbcf42318cb4ad7921 Mon Sep 17 00:00:00 2001 From: kettenis Date: Wed, 17 Feb 2021 12:11:44 +0000 Subject: [PATCH] Add support for FIQs. We need these to support agtimer(4) on Apple M1 SoCs since its interrupts seem to be hardwared to trigger an FIQ instead of an IRQ. This means we need to manipulate both the F and the I bit in the DAIF register when enabling and disabling interrupts. ok patrick@ --- sys/arch/arm64/arm64/exception.S | 37 ++++++++++++++++--- sys/arch/arm64/arm64/intr.c | 61 ++++++++++++++++++++++--------- sys/arch/arm64/arm64/trampoline.S | 8 +++- sys/arch/arm64/dev/agintc.c | 4 +- sys/arch/arm64/dev/ampintc.c | 4 +- sys/arch/arm64/dev/bcm2836_intr.c | 4 +- sys/arch/arm64/include/cpu.h | 16 ++++---- sys/arch/arm64/include/intr.h | 15 ++++---- sys/arch/arm64/include/profile.h | 4 +- 9 files changed, 104 insertions(+), 49 deletions(-) diff --git a/sys/arch/arm64/arm64/exception.S b/sys/arch/arm64/arm64/exception.S index 89fd88480e3..f63062991de 100644 --- a/sys/arch/arm64/arm64/exception.S +++ b/sys/arch/arm64/arm64/exception.S @@ -1,4 +1,4 @@ -/* $OpenBSD: exception.S,v 1.11 2020/03/17 17:27:12 kettenis Exp $ */ +/* $OpenBSD: exception.S,v 1.12 2021/02/17 12:11:44 kettenis Exp $ */ /*- * Copyright (c) 2014 Andrew Turner * All rights reserved. @@ -68,7 +68,7 @@ .macro restore_registers el .if \el == 1 - msr daifset, #2 + msr daifset, #3 /* * Disable interrupts, x18 may change in the interrupt exception * handler. For EL0 exceptions, do_ast already did this. @@ -116,7 +116,7 @@ /* Disable interrupts */ mrs x19, daif 1: - msr daifset, #2 + msr daifset, #3 /* Check for astpending */ mrs x18, tpidr_el1 @@ -171,7 +171,18 @@ handle_el1h_sync: handle_el1h_irq: save_registers 1 mov x0, sp - bl arm_cpu_intr + bl arm_cpu_irq + restore_registers 1 + eret + dsb nsh + isb + + .globl handle_el1h_fiq + .type handle_el1h_fiq,@function +handle_el1h_fiq: + save_registers 1 + mov x0, sp + bl arm_cpu_fiq restore_registers 1 eret dsb nsh @@ -210,7 +221,21 @@ handle_el0_irq: disable_ss bl _C_LABEL(vfp_save) mov x0, sp - bl arm_cpu_intr + bl arm_cpu_irq + do_ast + bl _C_LABEL(vfp_enable) + allow_ss + restore_registers 0 + return + + .globl handle_el0_fiq + .type handle_el0_fiq,@function +handle_el0_fiq: + save_registers 0 + disable_ss + bl _C_LABEL(vfp_save) + mov x0, sp + bl arm_cpu_fiq do_ast bl _C_LABEL(vfp_enable) allow_ss @@ -253,7 +278,7 @@ exception_vectors: vector el1h_sync /* Synchronous EL1h */ vector el1h_irq /* IRQ EL1h */ - vempty /* FIQ EL1h */ + vector el1h_fiq /* FIQ EL1h */ vector el1h_error /* Error EL1h */ vempty /* Synchronous 64-bit EL0 */ diff --git a/sys/arch/arm64/arm64/intr.c b/sys/arch/arm64/arm64/intr.c index 9d5514878c9..b840d3d85c9 100644 --- a/sys/arch/arm64/arm64/intr.c +++ b/sys/arch/arm64/arm64/intr.c @@ -1,4 +1,4 @@ -/* $OpenBSD: intr.c,v 1.19 2020/07/17 08:07:33 patrick Exp $ */ +/* $OpenBSD: intr.c,v 1.20 2021/02/17 12:11:44 kettenis Exp $ */ /* * Copyright (c) 2011 Dale Rahn * @@ -39,8 +39,11 @@ int arm_dflt_spllower(int); void arm_dflt_splx(int); void arm_dflt_setipl(int); -void arm_dflt_intr(void *); -void arm_cpu_intr(void *); +void arm_dflt_irq(void *); +void arm_dflt_fiq(void *); + +void arm_cpu_irq(void *); +void arm_cpu_fiq(void *); #define SI_TO_IRQBIT(x) (1 << (x)) uint32_t arm_smask[NIPL]; @@ -52,21 +55,40 @@ struct arm_intr_func arm_intr_func = { arm_dflt_setipl }; -void (*arm_intr_dispatch)(void *) = arm_dflt_intr; +void +arm_dflt_irq(void *frame) +{ + panic("%s", __func__); +} + +void +arm_dflt_fiq(void *frame) +{ + panic("%s", __func__); +} + +void (*arm_irq_dispatch)(void *) = arm_dflt_irq; void -arm_cpu_intr(void *frame) +arm_cpu_irq(void *frame) { struct cpu_info *ci = curcpu(); ci->ci_idepth++; - (*arm_intr_dispatch)(frame); + (*arm_irq_dispatch)(frame); ci->ci_idepth--; } + +void (*arm_fiq_dispatch)(void *) = arm_dflt_fiq; + void -arm_dflt_intr(void *frame) +arm_cpu_fiq(void *frame) { - panic("arm_dflt_intr() called"); + struct cpu_info *ci = curcpu(); + + ci->ci_idepth++; + (*arm_fiq_dispatch)(frame); + ci->ci_idepth--; } /* @@ -669,15 +691,20 @@ arm_do_pending_intr(int pcpl) restore_interrupts(oldirqstate); } -void arm_set_intr_handler(int (*raise)(int), int (*lower)(int), - void (*x)(int), void (*setipl)(int), - void (*intr_handle)(void *)) -{ - arm_intr_func.raise = raise; - arm_intr_func.lower = lower; - arm_intr_func.x = x; - arm_intr_func.setipl = setipl; - arm_intr_dispatch = intr_handle; +void +arm_set_intr_handler(int (*raise)(int), int (*lower)(int), + void (*x)(int), void (*setipl)(int), void (*irq_dispatch)(void *), + void (*fiq_dispatch)(void *)) +{ + arm_intr_func.raise = raise; + arm_intr_func.lower = lower; + arm_intr_func.x = x; + arm_intr_func.setipl = setipl; + + if (irq_dispatch) + arm_irq_dispatch = irq_dispatch; + if (fiq_dispatch) + arm_fiq_dispatch = fiq_dispatch; } void diff --git a/sys/arch/arm64/arm64/trampoline.S b/sys/arch/arm64/arm64/trampoline.S index 97240bb4e61..430db83e94d 100644 --- a/sys/arch/arm64/arm64/trampoline.S +++ b/sys/arch/arm64/arm64/trampoline.S @@ -1,4 +1,4 @@ -/* $OpenBSD: trampoline.S,v 1.2 2019/12/17 22:25:56 deraadt Exp $ */ +/* $OpenBSD: trampoline.S,v 1.3 2021/02/17 12:11:44 kettenis Exp $ */ /* * Copyright (c) 2018 Mark Kettenis @@ -63,7 +63,7 @@ trampoline_vectors: vector el0_sync /* Synchronous 64-bit EL0 */ vector el0_irq /* IRQ 64-bit EL0 */ - vempty /* FIQ 64-bit EL0 */ + vector el0_fiq /* FIQ 64-bit EL0 */ vector el0_error /* Error 64-bit EL0 */ vempty /* Synchronous 32-bit EL0 */ @@ -110,6 +110,10 @@ tramp_el0_irq: tramp_enter b handle_el0_irq +tramp_el0_fiq: + tramp_enter + b handle_el0_fiq + tramp_el0_error: tramp_enter b handle_el0_error diff --git a/sys/arch/arm64/dev/agintc.c b/sys/arch/arm64/dev/agintc.c index 2d1bab323d9..564f4d2ecd5 100644 --- a/sys/arch/arm64/dev/agintc.c +++ b/sys/arch/arm64/dev/agintc.c @@ -1,4 +1,4 @@ -/* $OpenBSD: agintc.c,v 1.29 2020/11/28 18:33:43 patrick Exp $ */ +/* $OpenBSD: agintc.c,v 1.30 2021/02/17 12:11:45 kettenis Exp $ */ /* * Copyright (c) 2007, 2009, 2011, 2017 Dale Rahn * Copyright (c) 2018 Mark Kettenis @@ -515,7 +515,7 @@ agintc_attach(struct device *parent, struct device *self, void *aux) /* insert self as interrupt handler */ arm_set_intr_handler(agintc_splraise, agintc_spllower, agintc_splx, - agintc_setipl, agintc_irq_handler); + agintc_setipl, agintc_irq_handler, NULL); /* enable interrupts */ ctrl = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR); diff --git a/sys/arch/arm64/dev/ampintc.c b/sys/arch/arm64/dev/ampintc.c index 3b9aee253af..7e1b8963bc5 100644 --- a/sys/arch/arm64/dev/ampintc.c +++ b/sys/arch/arm64/dev/ampintc.c @@ -1,4 +1,4 @@ -/* $OpenBSD: ampintc.c,v 1.20 2020/09/05 14:47:21 deraadt Exp $ */ +/* $OpenBSD: ampintc.c,v 1.21 2021/02/17 12:11:45 kettenis Exp $ */ /* * Copyright (c) 2007,2009,2011 Dale Rahn * @@ -302,7 +302,7 @@ ampintc_attach(struct device *parent, struct device *self, void *aux) /* insert self as interrupt handler */ arm_set_intr_handler(ampintc_splraise, ampintc_spllower, ampintc_splx, - ampintc_setipl, ampintc_irq_handler); + ampintc_setipl, ampintc_irq_handler, NULL); #ifdef MULTIPROCESSOR /* setup IPI interrupts */ diff --git a/sys/arch/arm64/dev/bcm2836_intr.c b/sys/arch/arm64/dev/bcm2836_intr.c index ed7295ae8ee..40b36fdc6ce 100644 --- a/sys/arch/arm64/dev/bcm2836_intr.c +++ b/sys/arch/arm64/dev/bcm2836_intr.c @@ -1,4 +1,4 @@ -/* $OpenBSD: bcm2836_intr.c,v 1.9 2020/07/14 15:34:14 patrick Exp $ */ +/* $OpenBSD: bcm2836_intr.c,v 1.10 2021/02/17 12:11:45 kettenis Exp $ */ /* * Copyright (c) 2007,2009 Dale Rahn * Copyright (c) 2015 Patrick Wildt @@ -211,7 +211,7 @@ bcm_intc_attach(struct device *parent, struct device *self, void *aux) /* insert self as interrupt handler */ arm_set_intr_handler(bcm_intc_splraise, bcm_intc_spllower, - bcm_intc_splx, bcm_intc_setipl, bcm_intc_irq_handler); + bcm_intc_splx, bcm_intc_setipl, bcm_intc_irq_handler, NULL); sc->sc_intc.ic_node = faa->fa_node; sc->sc_intc.ic_cookie = sc; diff --git a/sys/arch/arm64/include/cpu.h b/sys/arch/arm64/include/cpu.h index a4d18502997..f82d9fb5bcc 100644 --- a/sys/arch/arm64/include/cpu.h +++ b/sys/arch/arm64/include/cpu.h @@ -1,4 +1,4 @@ -/* $OpenBSD: cpu.h,v 1.18 2020/06/05 23:16:24 naddy Exp $ */ +/* $OpenBSD: cpu.h,v 1.19 2021/02/17 12:11:45 kettenis Exp $ */ /* * Copyright (c) 2016 Dale Rahn * @@ -256,7 +256,7 @@ void board_startup(void); // functions to manipulate interrupt state static __inline uint32_t -get_daif() +get_daif(void) { uint32_t daif; @@ -271,23 +271,23 @@ restore_daif(uint32_t daif) } static __inline void -enable_irq_daif() +enable_irq_daif(void) { - __asm volatile ("msr daifclr, #2"); + __asm volatile ("msr daifclr, #3"); } static __inline void -disable_irq_daif() +disable_irq_daif(void) { - __asm volatile ("msr daifset, #2"); + __asm volatile ("msr daifset, #3"); } static __inline uint32_t -disable_irq_daif_ret() +disable_irq_daif_ret(void) { uint32_t daif; __asm volatile ("mrs %x0, daif": "=r"(daif)); - __asm volatile ("msr daifset, #2"); + __asm volatile ("msr daifset, #3"); return daif; } diff --git a/sys/arch/arm64/include/intr.h b/sys/arch/arm64/include/intr.h index 7d299072ef7..d8c5b3640a0 100644 --- a/sys/arch/arm64/include/intr.h +++ b/sys/arch/arm64/include/intr.h @@ -1,4 +1,4 @@ -/* $OpenBSD: intr.h,v 1.16 2020/07/17 08:07:33 patrick Exp $ */ +/* $OpenBSD: intr.h,v 1.17 2021/02/17 12:11:45 kettenis Exp $ */ /* * Copyright (c) 2001-2004 Opsycon AB (www.opsycon.se / www.opsycon.com) @@ -81,14 +81,13 @@ #include #include -int splraise(int); -int spllower(int); -void splx(int); +int splraise(int); +int spllower(int); +void splx(int); -void arm_do_pending_intr(int); -void arm_set_intr_handler(int (*raise)(int), int (*lower)(int), - void (*x)(int), void (*setipl)(int), - void (*intr_handle)(void *)); +void arm_do_pending_intr(int); +void arm_set_intr_handler(int (*)(int), int (*)(int), void (*)(int), + void (*)(int), void (*)(void *), void (*)(void *)); struct arm_intr_handle { struct interrupt_controller *ih_ic; diff --git a/sys/arch/arm64/include/profile.h b/sys/arch/arm64/include/profile.h index 4e7d32336a1..1c53b7e6af1 100644 --- a/sys/arch/arm64/include/profile.h +++ b/sys/arch/arm64/include/profile.h @@ -1,4 +1,4 @@ -/* $OpenBSD: profile.h,v 1.1 2016/12/17 23:38:33 patrick Exp $ */ +/* $OpenBSD: profile.h,v 1.2 2021/02/17 12:11:45 kettenis Exp $ */ /* * Copyright (c) 2015 Dale Rahn * @@ -61,7 +61,7 @@ __asm__ (".text;" \ #ifdef _KERNEL // Change this to dair read/set, then restore. #define MCOUNT_ENTER \ -__asm__ ("mrs %x0,daif; msr daifset, #0x2": "=r"(s)); +__asm__ ("mrs %x0, daif; msr daifset, #0x3": "=r"(s)); #define MCOUNT_EXIT \ __asm__ ("msr daif, %x0":: "r"(s)); -- 2.20.1