Improve handling of SError interrupts. Print some useful information and

allow additional information to be printed for specific CPU types.  Use
this to print the L2C registers on Apple CPUs which can be very useful
in tracking down the source of certain SError interrupts.

ok miod@, dlg@

diff --git a/sys/arch/arm64/arm64/cpu.c b/sys/arch/arm64/arm64/cpu.c
index 3e40db9..ca530c0 100644 (file)
--- a/sys/arch/arm64/arm64/cpu.c
+++ b/sys/arch/arm64/arm64/cpu.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: cpu.c,v 1.101 2023/11/23 19:54:30 patrick Exp $       */
+/*     $OpenBSD: cpu.c,v 1.102 2023/12/26 09:19:15 kettenis Exp $      */
 
 /*
  * Copyright (c) 2016 Dale Rahn <drahn@dalerahn.com>
@@ -252,6 +252,7 @@ void        cpu_psci_init(struct cpu_info *);
 
 void   cpu_flush_bp_noop(void);
 void   cpu_flush_bp_psci(void);
+void   cpu_serror_apple(void);
 
 #if NKSTAT > 0
 void   cpu_kstat_attach(struct cpu_info *ci);
@@ -398,7 +399,6 @@ cpu_identify(struct cpu_info *ci)
         * The architecture has been updated to explicitly tell us if
         * we're not vulnerable to regular Spectre.
         */
-
        id = READ_SPECIALREG(id_aa64pfr0_el1);
        if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_IMPL)
                ci->ci_flush_bp = cpu_flush_bp_noop;
@@ -407,7 +407,6 @@ cpu_identify(struct cpu_info *ci)
         * But we might still be vulnerable to Spectre-BHB.  If we know the
         * CPU, we can add a branchy loop that cleans the BHB.
         */
-
        if (impl == CPU_IMPL_ARM) {
                switch (part) {
                case CPU_PART_CORTEX_A72:
@@ -453,13 +452,11 @@ cpu_identify(struct cpu_info *ci)
 #endif
 
        /* Prefer CLRBHB to mitigate Spectre-BHB. */
-
        id = READ_SPECIALREG(id_aa64isar2_el1);
        if (ID_AA64ISAR2_CLRBHB(id) >= ID_AA64ISAR2_CLRBHB_IMPL)
                ci->ci_trampoline_vectors = (vaddr_t)trampoline_vectors_clrbhb;
 
        /* ECBHB tells us Spectre-BHB is mitigated. */
-
        id = READ_SPECIALREG(id_aa64mmfr1_el1);
        if (ID_AA64MMFR1_ECBHB(id) >= ID_AA64MMFR1_ECBHB_IMPL)
                ci->ci_trampoline_vectors = (vaddr_t)trampoline_vectors_none;
@@ -468,13 +465,18 @@ cpu_identify(struct cpu_info *ci)
         * The architecture has been updated to explicitly tell us if
         * we're not vulnerable.
         */
-
        id = READ_SPECIALREG(id_aa64pfr0_el1);
        if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_HCXT) {
                ci->ci_flush_bp = cpu_flush_bp_noop;
                ci->ci_trampoline_vectors = (vaddr_t)trampoline_vectors_none;
        }
 
+       /*
+        * Apple CPUs provide detailed information for SError.
+        */
+       if (impl == CPU_IMPL_APPLE)
+               ci->ci_serror = cpu_serror_apple;
+
        /*
         * Print CPU features encoded in the ID registers.
         */
@@ -1018,6 +1020,15 @@ cpu_flush_bp_psci(void)
 #endif
 }
 
+void
+cpu_serror_apple(void)
+{
+       __asm volatile("dsb sy; isb" ::: "memory");
+       printf("l2c_err_sts 0x%llx\n", READ_SPECIALREG(s3_3_c15_c8_0));
+       printf("l2c_err_adr 0x%llx\n", READ_SPECIALREG(s3_3_c15_c9_0));
+       printf("l2c_err_inf 0x%llx\n", READ_SPECIALREG(s3_3_c15_c10_0));
+}
+
 int
 cpu_clockspeed(int *freq)
 {

diff --git a/sys/arch/arm64/arm64/exception.S b/sys/arch/arm64/arm64/exception.S
index 7192791..1a1ff5f 100644 (file)
--- a/sys/arch/arm64/arm64/exception.S
+++ b/sys/arch/arm64/arm64/exception.S
@@ -1,4 +1,4 @@
-/* $OpenBSD: exception.S,v 1.15 2022/12/10 10:13:58 patrick Exp $ */
+/* $OpenBSD: exception.S,v 1.16 2023/12/26 09:19:15 kettenis Exp $ */
 /*-
  * Copyright (c) 2014 Andrew Turner
  * All rights reserved.
@@ -191,7 +191,11 @@ handle_el1h_fiq:
        .globl handle_el1h_error
        .type handle_el1h_error,@function
 handle_el1h_error:
+       save_registers 1
+       mov     x0, sp
+       bl      do_el1h_error
        brk     0xf13
+       1: b 1b
 
 .macro return
        msr     tpidrro_el0, x18

diff --git a/sys/arch/arm64/arm64/trap.c b/sys/arch/arm64/arm64/trap.c
index c356163..bbe6b04 100644 (file)
--- a/sys/arch/arm64/arm64/trap.c
+++ b/sys/arch/arm64/arm64/trap.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: trap.c,v 1.46 2023/06/10 19:30:48 kettenis Exp $ */
+/* $OpenBSD: trap.c,v 1.47 2023/12/26 09:19:15 kettenis Exp $ */
 /*-
  * Copyright (c) 2014 Andrew Turner
  * All rights reserved.
@@ -338,12 +338,36 @@ do_el0_sync(struct trapframe *frame)
        userret(p);
 }
 
+static void
+serror(struct trapframe *frame)
+{
+       struct cpu_info *ci = curcpu();
+       uint64_t esr, far;
+
+       esr = READ_SPECIALREG(esr_el1);
+       far = READ_SPECIALREG(far_el1);
+
+       printf("SError: %lx esr %llx far %0llx\n",
+           frame->tf_elr, esr, far);
+
+       if (ci->ci_serror)
+               ci->ci_serror();
+}
+
 void
 do_el0_error(struct trapframe *frame)
 {
+       serror(frame);
        panic("do_el0_error");
 }
 
+void
+do_el1h_error(struct trapframe *frame)
+{
+       serror(frame);
+       panic("do_el1h_error");
+}
+
 void
 dumpregs(struct trapframe *frame)
 {

diff --git a/sys/arch/arm64/include/cpu.h b/sys/arch/arm64/include/cpu.h
index ec514d8..a457f27 100644 (file)
--- a/sys/arch/arm64/include/cpu.h
+++ b/sys/arch/arm64/include/cpu.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: cpu.h,v 1.39 2023/08/23 01:55:46 cheloha Exp $ */
+/* $OpenBSD: cpu.h,v 1.40 2023/12/26 09:19:15 kettenis Exp $ */
 /*
  * Copyright (c) 2016 Dale Rahn <drahn@dalerahn.com>
  *
@@ -143,6 +143,7 @@ struct cpu_info {
        int                     ci_want_resched;
 
        void                    (*ci_flush_bp)(void);
+       void                    (*ci_serror)(void);
 
        uint64_t                ci_ttbr1;
        vaddr_t                 ci_el1_stkend;