From: kettenis Date: Sat, 1 Jan 2022 18:52:36 +0000 (+0000) Subject: Rewrite the kernel FPU handling code. The new code saves the FPU state X-Git-Url: http://artulab.com/gitweb/?a=commitdiff_plain;h=b83a19dd9c2195f11121de7fba7a36fcd6a004f2;p=openbsd Rewrite the kernel FPU handling code. The new code saves the FPU state in cpu_switch() instead of at the kernel edge and gets rid of the FPU state tracking in struct cpu_info and struct pcb. This fixes the random crashes seen with SMP kernels on Apple M1. ok patrick@ --- diff --git a/sys/arch/arm64/arm64/cpuswitch.S b/sys/arch/arm64/arm64/cpuswitch.S index 0eaa6d583e3..9d53b26091b 100644 --- a/sys/arch/arm64/arm64/cpuswitch.S +++ b/sys/arch/arm64/arm64/cpuswitch.S @@ -1,4 +1,4 @@ -/* $OpenBSD: cpuswitch.S,v 1.4 2018/09/07 01:32:01 mortimer Exp $ */ +/* $OpenBSD: cpuswitch.S,v 1.5 2022/01/01 18:52:36 kettenis Exp $ */ /* * Copyright (c) 2015 Dale Rahn * @@ -27,7 +27,7 @@ * x4 - may contain user thread pointer (TCB) * x5 - may contain PCB pointer (new or old) */ -ENTRY(cpu_switchto) +ENTRY(cpu_switchto_asm) // check if old context needs to be saved cmp x0, #0 beq 1f @@ -46,18 +46,6 @@ ENTRY(cpu_switchto) ldr x5, [x2, #(CI_CURPCB)] str x3, [x5, #(PCB_SP) ] // save to old pcb - - mov x19, x1 //save new ctx across vfp - // old process has been saved - // save old fpu? - ldr w7, [x5, #(PCB_FLAGS)] - mov w3, #(PCB_FPU) - and w7, w7, w3 - cbz w7, 1f - - bl vfp_save - - mov x1, x19 1: RETGUARD_SYMBOL(cpu_switchto) RETGUARD_LOAD_RANDOM(cpu_switchto, x20) diff --git a/sys/arch/arm64/arm64/cryptox.c b/sys/arch/arm64/arm64/cryptox.c index 02c93751bc7..494c59af5ae 100644 --- a/sys/arch/arm64/arm64/cryptox.c +++ b/sys/arch/arm64/arm64/cryptox.c @@ -1,4 +1,4 @@ -/* $OpenBSD: cryptox.c,v 1.5 2021/10/24 10:26:22 patrick Exp $ */ +/* $OpenBSD: cryptox.c,v 1.6 2022/01/01 18:52:36 kettenis Exp $ */ /* * Copyright (c) 2003 Jason Wright * Copyright (c) 2003, 2004 Theo de Raadt @@ -34,7 +34,7 @@ #include #include -#include +#include struct cryptox_aes_key { uint32_t rd_key[4 *(AES_MAXROUNDS + 1)]; @@ -163,10 +163,10 @@ cryptox_newsession(u_int32_t *sidp, struct cryptoini *cri) switch (c->cri_alg) { case CRYPTO_AES_CBC: ses->ses_klen = c->cri_klen / 8; - vfp_kernel_enter(); + fpu_kernel_enter(); aes_v8_set_encrypt_key(c->cri_key, c->cri_klen, &ses->ses_ekey); aes_v8_set_decrypt_key(c->cri_key, c->cri_klen, &ses->ses_dkey); - vfp_kernel_exit(); + fpu_kernel_exit(); break; case CRYPTO_MD5_HMAC: @@ -410,7 +410,7 @@ cryptox_encdec(struct cryptop *crp, struct cryptodesc *crd, crd->crd_len, buf); /* Apply cipher */ - vfp_kernel_enter(); + fpu_kernel_enter(); switch (crd->crd_alg) { case CRYPTO_AES_CBC: if (crd->crd_flags & CRD_F_ENCRYPT) @@ -419,7 +419,7 @@ cryptox_encdec(struct cryptop *crp, struct cryptodesc *crd, aes_v8_cbc_encrypt(buf, buf, crd->crd_len, &ses->ses_dkey, iv, 0); break; } - vfp_kernel_exit(); + fpu_kernel_exit(); cryptox_ops++; diff --git a/sys/arch/arm64/arm64/exception.S b/sys/arch/arm64/arm64/exception.S index f63062991de..e28430e58ba 100644 --- a/sys/arch/arm64/arm64/exception.S +++ b/sys/arch/arm64/arm64/exception.S @@ -1,4 +1,4 @@ -/* $OpenBSD: exception.S,v 1.12 2021/02/17 12:11:44 kettenis Exp $ */ +/* $OpenBSD: exception.S,v 1.13 2022/01/01 18:52:36 kettenis Exp $ */ /*- * Copyright (c) 2014 Andrew Turner * All rights reserved. @@ -209,7 +209,6 @@ handle_el0_sync: mov x0, sp bl do_el0_sync do_ast - bl _C_LABEL(vfp_enable) allow_ss restore_registers 0 return @@ -219,11 +218,9 @@ handle_el0_sync: handle_el0_irq: save_registers 0 disable_ss - bl _C_LABEL(vfp_save) mov x0, sp bl arm_cpu_irq do_ast - bl _C_LABEL(vfp_enable) allow_ss restore_registers 0 return @@ -233,11 +230,9 @@ handle_el0_irq: handle_el0_fiq: save_registers 0 disable_ss - bl _C_LABEL(vfp_save) mov x0, sp bl arm_cpu_fiq do_ast - bl _C_LABEL(vfp_enable) allow_ss restore_registers 0 return diff --git a/sys/arch/arm64/arm64/fpu.c b/sys/arch/arm64/arm64/fpu.c new file mode 100644 index 00000000000..0285ef78558 --- /dev/null +++ b/sys/arch/arm64/arm64/fpu.c @@ -0,0 +1,187 @@ +/* $OpenBSD: fpu.c,v 1.1 2022/01/01 18:52:36 kettenis Exp $ */ +/* + * Copyright (c) 2022 Mark Kettenis + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include +#include +#include + +#include + +void +fpu_save(struct proc *p) +{ + struct pcb *pcb = &p->p_addr->u_pcb; + struct fpreg *fp = &pcb->pcb_fpstate; + uint64_t cpacr; + + cpacr = READ_SPECIALREG(cpacr_el1); + if ((cpacr & CPACR_FPEN_MASK) == CPACR_FPEN_TRAP_ALL1) + return; + KASSERT((cpacr & CPACR_FPEN_MASK) == CPACR_FPEN_TRAP_NONE); + +#define STRQx(x) \ + __asm volatile ("str q" #x ", [%0, %1]" :: "r"(fp->fp_reg), "i"(x * 16)) + + STRQx(0); + STRQx(1); + STRQx(2); + STRQx(3); + STRQx(4); + STRQx(5); + STRQx(6); + STRQx(7); + STRQx(8); + STRQx(9); + STRQx(10); + STRQx(11); + STRQx(12); + STRQx(13); + STRQx(14); + STRQx(15); + STRQx(16); + STRQx(17); + STRQx(18); + STRQx(19); + STRQx(20); + STRQx(21); + STRQx(22); + STRQx(23); + STRQx(24); + STRQx(25); + STRQx(26); + STRQx(27); + STRQx(28); + STRQx(29); + STRQx(30); + STRQx(31); + + fp->fp_sr = READ_SPECIALREG(fpsr); + fp->fp_cr = READ_SPECIALREG(fpcr); +} + +void +fpu_load(struct proc *p) +{ + struct pcb *pcb = &p->p_addr->u_pcb; + struct fpreg *fp = &pcb->pcb_fpstate; + uint64_t cpacr; + + cpacr = READ_SPECIALREG(cpacr_el1); + KASSERT((cpacr & CPACR_FPEN_MASK) == CPACR_FPEN_TRAP_ALL1); + + if ((pcb->pcb_flags & PCB_FPU) == 0) { + memset(fp, 0, sizeof(*fp)); + pcb->pcb_flags |= PCB_FPU; + } + + /* Enable FPU. */ + cpacr &= ~CPACR_FPEN_MASK; + cpacr |= CPACR_FPEN_TRAP_NONE; + WRITE_SPECIALREG(cpacr_el1, cpacr); + __asm volatile ("isb"); + +#define LDRQx(x) \ + __asm volatile ("ldr q" #x ", [%0, %1]" :: "r"(fp->fp_reg), "i"(x * 16)) + + LDRQx(0); + LDRQx(1); + LDRQx(2); + LDRQx(3); + LDRQx(4); + LDRQx(5); + LDRQx(6); + LDRQx(7); + LDRQx(8); + LDRQx(9); + LDRQx(10); + LDRQx(11); + LDRQx(12); + LDRQx(13); + LDRQx(14); + LDRQx(15); + LDRQx(16); + LDRQx(17); + LDRQx(18); + LDRQx(19); + LDRQx(20); + LDRQx(21); + LDRQx(22); + LDRQx(23); + LDRQx(24); + LDRQx(25); + LDRQx(26); + LDRQx(27); + LDRQx(28); + LDRQx(29); + LDRQx(30); + LDRQx(31); + + WRITE_SPECIALREG(fpsr, fp->fp_sr); + WRITE_SPECIALREG(fpcr, fp->fp_cr); +} + +void +fpu_drop(void) +{ + uint64_t cpacr; + + /* Disable FPU. */ + cpacr = READ_SPECIALREG(cpacr_el1); + cpacr &= ~CPACR_FPEN_MASK; + cpacr |= CPACR_FPEN_TRAP_ALL1; + WRITE_SPECIALREG(cpacr_el1, cpacr); + + /* + * No ISB instruction needed here, as returning to EL0 is a + * context synchronization event. + */ +} + +void +fpu_kernel_enter(void) +{ + struct pcb *pcb = &curproc->p_addr->u_pcb; + uint64_t cpacr; + + if (pcb->pcb_flags & PCB_FPU) + fpu_save(curproc); + + /* Enable FPU (kernel only). */ + cpacr = READ_SPECIALREG(cpacr_el1); + cpacr &= ~CPACR_FPEN_MASK; + cpacr |= CPACR_FPEN_TRAP_EL0; + WRITE_SPECIALREG(cpacr_el1, cpacr); + __asm volatile ("isb"); +} + +void +fpu_kernel_exit(void) +{ + uint64_t cpacr; + + /* Disable FPU. */ + cpacr = READ_SPECIALREG(cpacr_el1); + cpacr &= ~CPACR_FPEN_MASK; + cpacr |= CPACR_FPEN_TRAP_ALL1; + WRITE_SPECIALREG(cpacr_el1, cpacr); + + /* + * No ISB instruction needed here, as returning to EL0 is a + * context synchronization event. + */ +} diff --git a/sys/arch/arm64/arm64/machdep.c b/sys/arch/arm64/arm64/machdep.c index f4ad2f8b0ab..4b20d6528c6 100644 --- a/sys/arch/arm64/arm64/machdep.c +++ b/sys/arch/arm64/arm64/machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: machdep.c,v 1.66 2021/12/06 09:49:46 jsg Exp $ */ +/* $OpenBSD: machdep.c,v 1.67 2022/01/01 18:52:36 kettenis Exp $ */ /* * Copyright (c) 2014 Patrick Wildt * Copyright (c) 2021 Mark Kettenis @@ -43,7 +43,7 @@ #include #include #include -#include +#include #include #include @@ -294,6 +294,23 @@ cpu_startup(void) } } +void cpu_switchto_asm(struct proc *, struct proc *); + +void +cpu_switchto(struct proc *old, struct proc *new) +{ + if (old) { + struct pcb *pcb = &old->p_addr->u_pcb; + + if (pcb->pcb_flags & PCB_FPU) + fpu_save(old); + + fpu_drop(); + } + + cpu_switchto_asm(old, new); +} + /* * machine dependent system variables. */ @@ -395,14 +412,13 @@ void setregs(struct proc *p, struct exec_package *pack, u_long stack, register_t *retval) { - struct trapframe *tf; + struct pcb *pcb = &p->p_addr->u_pcb; + struct trapframe *tf = pcb->pcb_tf; /* If we were using the FPU, forget about it. */ - if (p->p_addr->u_pcb.pcb_fpcpu != NULL) - vfp_discard(p); - p->p_addr->u_pcb.pcb_flags &= ~PCB_FPU; - - tf = p->p_addr->u_pcb.pcb_tf; + memset(&pcb->pcb_fpstate, 0, sizeof(pcb->pcb_fpstate)); + pcb->pcb_flags &= ~PCB_FPU; + fpu_drop(); memset (tf,0, sizeof(*tf)); tf->tf_sp = stack; diff --git a/sys/arch/arm64/arm64/process_machdep.c b/sys/arch/arm64/arm64/process_machdep.c index 0d19fe8b3d3..8703a9178ae 100644 --- a/sys/arch/arm64/arm64/process_machdep.c +++ b/sys/arch/arm64/arm64/process_machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: process_machdep.c,v 1.5 2018/08/03 18:36:01 kettenis Exp $ */ +/* $OpenBSD: process_machdep.c,v 1.6 2022/01/01 18:52:36 kettenis Exp $ */ /* * Copyright (c) 2014 Patrick Wildt * @@ -45,6 +45,7 @@ #include #include +#include #include #include @@ -68,6 +69,9 @@ process_read_regs(struct proc *p, struct reg *regs) int process_read_fpregs(struct proc *p, struct fpreg *regs) { + if (p->p_addr->u_pcb.pcb_flags & PCB_FPU) + fpu_save(p); + if (p->p_addr->u_pcb.pcb_flags & PCB_FPU) memcpy(regs, &p->p_addr->u_pcb.pcb_fpstate, sizeof(*regs)); else @@ -95,9 +99,13 @@ process_write_regs(struct proc *p, struct reg *regs) int process_write_fpregs(struct proc *p, struct fpreg *regs) { - p->p_addr->u_pcb.pcb_flags |= PCB_FPU; memcpy(&p->p_addr->u_pcb.pcb_fpstate, regs, sizeof(p->p_addr->u_pcb.pcb_fpstate)); + p->p_addr->u_pcb.pcb_flags |= PCB_FPU; + + /* drop FPU state */ + fpu_drop(); + return(0); } diff --git a/sys/arch/arm64/arm64/syscall.c b/sys/arch/arm64/arm64/syscall.c index a222c8410b0..e30e6fd0671 100644 --- a/sys/arch/arm64/arm64/syscall.c +++ b/sys/arch/arm64/arm64/syscall.c @@ -1,4 +1,4 @@ -/* $OpenBSD: syscall.c,v 1.9 2021/12/09 00:26:11 guenther Exp $ */ +/* $OpenBSD: syscall.c,v 1.10 2022/01/01 18:52:36 kettenis Exp $ */ /* * Copyright (c) 2015 Dale Rahn * @@ -28,8 +28,6 @@ #include -#include - #define MAXARGS 8 void @@ -43,9 +41,6 @@ svc_handler(trapframe_t *frame) uvmexp.syscalls++; - /* Before enabling interrupts, save FPU state */ - vfp_save(); - /* Re-enable interrupts if they were enabled previously */ if (__predict_true((frame->tf_spsr & I_bit) == 0)) intr_enable(); diff --git a/sys/arch/arm64/arm64/trap.c b/sys/arch/arm64/arm64/trap.c index d8cdf13ced9..e859b0cdd72 100644 --- a/sys/arch/arm64/arm64/trap.c +++ b/sys/arch/arm64/arm64/trap.c @@ -1,4 +1,4 @@ -/* $OpenBSD: trap.c,v 1.38 2021/05/16 15:10:19 deraadt Exp $ */ +/* $OpenBSD: trap.c,v 1.39 2022/01/01 18:52:36 kettenis Exp $ */ /*- * Copyright (c) 2014 Andrew Turner * All rights reserved. @@ -43,12 +43,12 @@ #include #include +#include +#include #include #include -#include #include -#include #ifdef KDB #include @@ -223,7 +223,7 @@ do_el1h_sync(struct trapframe *frame) switch(exception) { case EXCP_FP_SIMD: case EXCP_TRAP_FP: - panic("VFP exception in the kernel"); + panic("FP exception in the kernel"); case EXCP_DATA_ABORT: kdata_abort(frame, esr, far, 0); break; @@ -273,46 +273,38 @@ do_el0_sync(struct trapframe *frame) switch (exception) { case EXCP_UNKNOWN: - vfp_save(); curcpu()->ci_flush_bp(); sv.sival_ptr = (void *)frame->tf_elr; trapsignal(p, SIGILL, 0, ILL_ILLOPC, sv); break; case EXCP_FP_SIMD: case EXCP_TRAP_FP: - vfp_fault(frame->tf_elr, 0, frame, exception); + fpu_load(p); break; case EXCP_SVC: - vfp_save(); svc_handler(frame); break; case EXCP_INSN_ABORT_L: - vfp_save(); udata_abort(frame, esr, far, 1); break; case EXCP_PC_ALIGN: - vfp_save(); curcpu()->ci_flush_bp(); sv.sival_ptr = (void *)frame->tf_elr; trapsignal(p, SIGBUS, 0, BUS_ADRALN, sv); break; case EXCP_SP_ALIGN: - vfp_save(); curcpu()->ci_flush_bp(); sv.sival_ptr = (void *)frame->tf_sp; trapsignal(p, SIGBUS, 0, BUS_ADRALN, sv); break; case EXCP_DATA_ABORT_L: - vfp_save(); udata_abort(frame, esr, far, 0); break; case EXCP_BRK: - vfp_save(); sv.sival_ptr = (void *)frame->tf_elr; trapsignal(p, SIGTRAP, 0, TRAP_BRKPT, sv); break; case EXCP_SOFTSTP_EL0: - vfp_save(); sv.sival_ptr = (void *)frame->tf_elr; trapsignal(p, SIGTRAP, 0, TRAP_TRACE, sv); break; diff --git a/sys/arch/arm64/arm64/vfp.c b/sys/arch/arm64/arm64/vfp.c deleted file mode 100644 index 8c8521b9564..00000000000 --- a/sys/arch/arm64/arm64/vfp.c +++ /dev/null @@ -1,270 +0,0 @@ -/* $OpenBSD: vfp.c,v 1.6 2021/06/29 19:58:21 kettenis Exp $ */ -/* - * Copyright (c) 2011 Dale Rahn - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include -#include -#include -#include - -#include - -static inline void -set_vfp_enable(int val) -{ - uint64_t v; - __asm __volatile("mrs %x0, cpacr_el1" : "=r" (v)); - if (val != 0) { - v |= VFP_UFPEN; - } else { - v &= ~(VFP_UFPEN); - } - __asm __volatile("msr cpacr_el1, %x0" :: "r" (v)); - __asm __volatile("isb"); -} - -static inline int -get_vfp_enable(void) -{ - uint64_t v; - int enabled = 0; - __asm __volatile("mrs %x0, cpacr_el1" : "=r" (v)); - if ((v & VFP_UFPEN) == VFP_UFPEN ) - enabled = 1; - return enabled; -} - -int vfp_fault(vaddr_t pc, uint32_t insn, trapframe_t *tf, int fault_code); -void vfp_load(struct proc *p); -void vfp_store(struct fpreg *vfpsave); - -void -vfp_store(struct fpreg *vfpsave) -{ - uint32_t scratch; - - if (get_vfp_enable()) { - __asm __volatile( - "str q0, [%x1]\n" - "str q1, [%x1, #0x10]\n" - "str q2, [%x1, #0x20]\n" - "str q3, [%x1, #0x30]\n" - "str q4, [%x1, #0x40]\n" - "str q5, [%x1, #0x50]\n" - "str q6, [%x1, 0x60]\n" - "str q7, [%x1, #0x70]\n" - "str q8, [%x1, #0x80]\n" - "str q9, [%x1, #0x90]\n" - "str q10, [%x1, #0xa0]\n" - "str q11, [%x1, #0xb0]\n" - "str q12, [%x1, #0xc0]\n" - "str q13, [%x1, 0xd0]\n" - "str q14, [%x1, #0xe0]\n" - "str q15, [%x1, #0xf0]\n" - "str q16, [%x1, #0x100]\n" - "str q17, [%x1, #0x110]\n" - "str q18, [%x1, #0x120]\n" - "str q19, [%x1, #0x130]\n" - "str q20, [%x1, #0x140]\n" - "str q21, [%x1, #0x150]\n" - "str q22, [%x1, #0x160]\n" - "str q23, [%x1, #0x170]\n" - "str q24, [%x1, #0x180]\n" - "str q25, [%x1, #0x190]\n" - "str q26, [%x1, #0x1a0]\n" - "str q27, [%x1, #0x1b0]\n" - "str q28, [%x1, #0x1c0]\n" - "str q29, [%x1, #0x1d0]\n" - "str q30, [%x1, #0x1e0]\n" - "str q31, [%x1, #0x1f0]\n" - "mrs %x0, fpsr\n" - "str %w0, [%x1, 0x200]\n" /* save vfpscr */ - "mrs %x0, fpcr\n" - "str %w0, [%x1, 0x204]\n" /* save vfpscr */ - : "=&r" (scratch) : "r" (vfpsave)); - } - - /* disable FPU */ - set_vfp_enable(0); -} - -void -vfp_save(void) -{ - struct cpu_info *ci = curcpu(); - struct pcb *pcb = curpcb; - struct proc *p = curproc; - uint32_t vfp_enabled; - - if (ci->ci_fpuproc == 0) - return; - - vfp_enabled = get_vfp_enable(); - - if (!vfp_enabled) - return; /* not enabled, nothing to do */ - - if (pcb->pcb_fpcpu == NULL || ci->ci_fpuproc == NULL || - !(pcb->pcb_fpcpu == ci && ci->ci_fpuproc == p)) { - /* disable fpu before panic, otherwise recurse */ - set_vfp_enable(0); - - panic("FPU unit enabled when curproc and curcpu dont agree %p %p %p %p", pcb->pcb_fpcpu, ci, ci->ci_fpuproc, p); - } - - vfp_store(&p->p_addr->u_pcb.pcb_fpstate); - - /* - * NOTE: fpu state is saved but remains 'valid', as long as - * curpcb()->pcb_fpucpu == ci && ci->ci_fpuproc == curproc() - * is true FPU state is valid and can just be enabled without reload. - */ - set_vfp_enable(0); -} - -void -vfp_enable(void) -{ - struct cpu_info *ci = curcpu(); - - if (curproc->p_addr->u_pcb.pcb_fpcpu == ci && - ci->ci_fpuproc == curproc) { - intr_disable(); - - /* FPU state is still valid, just enable and go */ - set_vfp_enable(1); - } -} - -void -vfp_load(struct proc *p) -{ - struct cpu_info *ci = curcpu(); - struct pcb *pcb = &p->p_addr->u_pcb; - uint32_t scratch = 0; - u_long psw; - - /* do not allow a partially synced state here */ - psw = intr_disable(); - - /* - * p->p_pcb->pcb_fpucpu _may_ not be NULL here, but the FPU state - * was synced on kernel entry, so we can steal the FPU state - * instead of signalling and waiting for it to save - */ - - /* enable to be able to load ctx */ - set_vfp_enable(1); - - __asm __volatile( - "ldr q0, [%x1]\n" - "ldr q1, [%x1, #0x10]\n" - "ldr q2, [%x1, #0x20]\n" - "ldr q3, [%x1, #0x30]\n" - "ldr q4, [%x1, #0x40]\n" - "ldr q5, [%x1, #0x50]\n" - "ldr q6, [%x1, 0x60]\n" - "ldr q7, [%x1, #0x70]\n" - "ldr q8, [%x1, #0x80]\n" - "ldr q9, [%x1, #0x90]\n" - "ldr q10, [%x1, #0xa0]\n" - "ldr q11, [%x1, #0xb0]\n" - "ldr q12, [%x1, #0xc0]\n" - "ldr q13, [%x1, 0xd0]\n" - "ldr q14, [%x1, #0xe0]\n" - "ldr q15, [%x1, #0xf0]\n" - "ldr q16, [%x1, #0x100]\n" - "ldr q17, [%x1, #0x110]\n" - "ldr q18, [%x1, #0x120]\n" - "ldr q19, [%x1, #0x130]\n" - "ldr q20, [%x1, #0x140]\n" - "ldr q21, [%x1, #0x150]\n" - "ldr q22, [%x1, #0x160]\n" - "ldr q23, [%x1, #0x170]\n" - "ldr q24, [%x1, #0x180]\n" - "ldr q25, [%x1, #0x190]\n" - "ldr q26, [%x1, #0x1a0]\n" - "ldr q27, [%x1, #0x1b0]\n" - "ldr q28, [%x1, #0x1c0]\n" - "ldr q29, [%x1, #0x1d0]\n" - "ldr q30, [%x1, #0x1e0]\n" - "ldr q31, [%x1, #0x1f0]\n" - - "ldr %w0, [%x1, #0x200]\n" /* set old fpsr */ - "msr fpsr, %x0\n" - "ldr %w0, [%x1, #0x204]\n" /* set old fpsr */ - "msr fpcr, %x0\n" - : "=&r" (scratch) : "r" (&pcb->pcb_fpstate)); - - ci->ci_fpuproc = p; - pcb->pcb_fpcpu = ci; - - /* disable until return to userland */ - set_vfp_enable(0); - - intr_restore(psw); -} - - -int -vfp_fault(vaddr_t pc, uint32_t insn, trapframe_t *tf, int fault_code) -{ - struct proc *p = curproc; - struct pcb *pcb = &p->p_addr->u_pcb; - - if (get_vfp_enable()) { - /* - * We probably ran into an unsupported instruction, - * like NEON on a non-NEON system. Let the process know. - */ - return 1; - } - - /* we should be able to ignore old state of pcb_fpcpu ci_fpuproc */ - if ((pcb->pcb_flags & PCB_FPU) == 0) { - pcb->pcb_flags |= PCB_FPU; - memset(&pcb->pcb_fpstate, 0, sizeof (pcb->pcb_fpstate)); - } - vfp_load(p); - - return 0; -} - -void -vfp_discard(struct proc *p) -{ - struct cpu_info *ci = curcpu(); - - if (curpcb->pcb_fpcpu == ci && ci->ci_fpuproc == p) { - ci->ci_fpuproc = NULL; - curpcb->pcb_fpcpu = NULL; - } -} - -void -vfp_kernel_enter(void) -{ - struct cpu_info *ci = curcpu(); - - ci->ci_fpuproc = NULL; - set_vfp_enable(1); -} - -void -vfp_kernel_exit(void) -{ - set_vfp_enable(0); -} diff --git a/sys/arch/arm64/arm64/vm_machdep.c b/sys/arch/arm64/arm64/vm_machdep.c index 1a55d621606..1cce1ed5867 100644 --- a/sys/arch/arm64/arm64/vm_machdep.c +++ b/sys/arch/arm64/arm64/vm_machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: vm_machdep.c,v 1.8 2021/05/16 06:20:29 jsg Exp $ */ +/* $OpenBSD: vm_machdep.c,v 1.9 2022/01/01 18:52:36 kettenis Exp $ */ /* $NetBSD: vm_machdep.c,v 1.1 2003/04/26 18:39:33 fvdl Exp $ */ /*- @@ -56,8 +56,8 @@ #include #include +#include #include -#include /* * Finish a fork operation, with process p2 nearly set up. @@ -70,14 +70,16 @@ cpu_fork(struct proc *p1, struct proc *p2, void *stack, void *tcb, void (*func)(void *), void *arg) { struct pcb *pcb = &p2->p_addr->u_pcb; + struct pcb *pcb1 = &p1->p_addr->u_pcb; struct trapframe *tf; struct switchframe *sf; - // Does any flushing need to be done if process was running? + /* Save FPU state to PCB if necessary. */ + if (pcb1->pcb_flags & PCB_FPU) + fpu_save(p1); /* Copy the pcb. */ *pcb = p1->p_addr->u_pcb; - pcb->pcb_fpcpu = NULL; tf = (struct trapframe *)((u_long)p2->p_addr + USPACE @@ -109,10 +111,6 @@ cpu_fork(struct proc *p1, struct proc *p2, void *stack, void *tcb, void cpu_exit(struct proc *p) { - /* If we were using the FPU, forget about it. */ - if (p->p_addr->u_pcb.pcb_fpcpu != NULL) - vfp_discard(p); - pmap_deactivate(p); sched_exit(p); } diff --git a/sys/arch/arm64/conf/files.arm64 b/sys/arch/arm64/conf/files.arm64 index 574cedea6ca..18ef641e72a 100644 --- a/sys/arch/arm64/conf/files.arm64 +++ b/sys/arch/arm64/conf/files.arm64 @@ -1,4 +1,4 @@ -# $OpenBSD: files.arm64,v 1.50 2021/12/24 00:01:39 patrick Exp $ +# $OpenBSD: files.arm64,v 1.51 2022/01/01 18:52:37 kettenis Exp $ maxpartitions 16 maxusers 2 8 128 @@ -25,9 +25,9 @@ file arch/arm64/arm64/syscall.c file arch/arm64/arm64/sys_machdep.c file arch/arm64/arm64/cpu.c +file arch/arm64/arm64/fpu.c file arch/arm64/arm64/intr.c file arch/arm64/arm64/softintr.c -file arch/arm64/arm64/vfp.c file arch/arm64/arm64/exception.S file arch/arm64/arm64/trampoline.S file arch/arm64/arm64/trap.c diff --git a/sys/arch/arm64/dev/efi.c b/sys/arch/arm64/dev/efi.c index 078ab74a163..a4b41b0a97e 100644 --- a/sys/arch/arm64/dev/efi.c +++ b/sys/arch/arm64/dev/efi.c @@ -1,4 +1,4 @@ -/* $OpenBSD: efi.c,v 1.11 2021/10/24 17:52:28 mpi Exp $ */ +/* $OpenBSD: efi.c,v 1.12 2022/01/01 18:52:37 kettenis Exp $ */ /* * Copyright (c) 2017 Mark Kettenis @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include @@ -232,7 +232,7 @@ efi_enter(struct efi_softc *sc) __asm volatile("isb"); cpu_setttb(pm->pm_asid, pm->pm_pt0pa); - vfp_kernel_enter(); + fpu_kernel_enter(); } void @@ -240,7 +240,7 @@ efi_leave(struct efi_softc *sc) { struct pmap *pm = curcpu()->ci_curpm; - vfp_kernel_exit(); + fpu_kernel_exit(); WRITE_SPECIALREG(ttbr0_el1, pmap_kernel()->pm_pt0pa); __asm volatile("isb"); diff --git a/sys/arch/arm64/include/armreg.h b/sys/arch/arm64/include/armreg.h index fe96344af2a..51d0373c9bf 100644 --- a/sys/arch/arm64/include/armreg.h +++ b/sys/arch/arm64/include/armreg.h @@ -1,4 +1,4 @@ -/* $OpenBSD: armreg.h,v 1.17 2021/09/02 10:48:52 kettenis Exp $ */ +/* $OpenBSD: armreg.h,v 1.18 2022/01/01 18:52:37 kettenis Exp $ */ /*- * Copyright (c) 2013, 2014 Andrew Turner * Copyright (c) 2015 The FreeBSD Foundation @@ -159,7 +159,7 @@ #define ESR_ELx_EC_MASK (0x3f << 26) #define ESR_ELx_EXCEPTION(esr) (((esr) & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT) #define EXCP_UNKNOWN 0x00 /* Unkwn exception */ -#define EXCP_FP_SIMD 0x07 /* VFP/SIMD trap */ +#define EXCP_FP_SIMD 0x07 /* FP/SIMD trap */ #define EXCP_ILL_STATE 0x0e /* Illegal execution state */ #define EXCP_SVC 0x15 /* SVC trap */ #define EXCP_MSR 0x18 /* MSR/MRS trap */ diff --git a/sys/arch/arm64/include/cpu.h b/sys/arch/arm64/include/cpu.h index 109dd9c1141..c09ff800e25 100644 --- a/sys/arch/arm64/include/cpu.h +++ b/sys/arch/arm64/include/cpu.h @@ -1,4 +1,4 @@ -/* $OpenBSD: cpu.h,v 1.23 2021/11/22 19:22:59 kettenis Exp $ */ +/* $OpenBSD: cpu.h,v 1.24 2022/01/01 18:52:37 kettenis Exp $ */ /* * Copyright (c) 2016 Dale Rahn * @@ -99,7 +99,6 @@ struct cpu_info { struct proc *ci_curproc; struct pmap *ci_curpm; - struct proc *ci_fpuproc; u_int32_t ci_randseed; struct pcb *ci_curpcb; diff --git a/sys/arch/arm64/include/fpu.h b/sys/arch/arm64/include/fpu.h new file mode 100644 index 00000000000..74faa8dc0ba --- /dev/null +++ b/sys/arch/arm64/include/fpu.h @@ -0,0 +1,28 @@ +/* $OpenBSD: fpu.h,v 1.1 2022/01/01 18:52:37 kettenis Exp $ */ +/* + * Copyright (c) 2022 Mark Kettenis + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef _MACHINE_FPU_H +#define _MACHINE_FPU_H + +void fpu_save(struct proc *); +void fpu_load(struct proc *); +void fpu_drop(void); + +void fpu_kernel_enter(void); +void fpu_kernel_exit(void); + +#endif /* !_MACHINE_FPU_H */ diff --git a/sys/arch/arm64/include/pcb.h b/sys/arch/arm64/include/pcb.h index d56a1c1f7ab..968236180cc 100644 --- a/sys/arch/arm64/include/pcb.h +++ b/sys/arch/arm64/include/pcb.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pcb.h,v 1.4 2018/07/04 17:52:29 drahn Exp $ */ +/* $OpenBSD: pcb.h,v 1.5 2022/01/01 18:52:37 kettenis Exp $ */ /* * Copyright (c) 2016 Dale Rahn * @@ -38,7 +38,6 @@ struct pcb { caddr_t pcb_onfault; // On fault handler struct fpreg pcb_fpstate; // Floating Point state */ - struct cpu_info *pcb_fpcpu; void *pcb_tcb; }; diff --git a/sys/arch/arm64/include/vfp.h b/sys/arch/arm64/include/vfp.h deleted file mode 100644 index cc060163130..00000000000 --- a/sys/arch/arm64/include/vfp.h +++ /dev/null @@ -1,52 +0,0 @@ -/* $OpenBSD: vfp.h,v 1.4 2018/07/02 07:23:37 kettenis Exp $ */ -/*- - * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. - * - * This software was developed by Andrew Turner under - * sponsorship from the FreeBSD Foundation. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD: head/sys/arm64/include/vfp.h 281494 2015-04-13 14:43:10Z andrew $ - */ - -#ifndef _MACHINE_VFP_H_ -#define _MACHINE_VFP_H_ - -#ifdef _KERNEL - -#define VFP_KFPEN (1 << 20) -#define VFP_UFPEN (3 << 20) - -#ifndef LOCORE -void vfp_discard(struct proc *); -void vfp_save(void); -void vfp_enable(void); -int vfp_fault(vaddr_t, uint32_t, trapframe_t *, int); -void vfp_kernel_enter(void); -void vfp_kernel_exit(void); -#endif - -#endif - -#endif /* !_MACHINE_VFP_H_ */