On CPUs with eIBRS ("enhanced Indirect Branch Restricted Speculation")

or IBT enabled the kernel, the hardware should the attacks which
retpolines were created to prevent.  In those cases, retpolines
should be a net negative for security as they are an indirect branch
gadget.  They're also slower.
 * use -mretpoline-external-thunk to give us control of the code
   used for indirect branches
 * default to using a retpoline as before, but marks it and the
   other ASM kernel retpolines for code patching
 * if the CPU has eIBRS, then enable it
 * if the CPU has eIBRS *or* IBT, then codepatch the three different
   retpolines to just indirect jumps

make clean && make config required after this

ok kettenis@

diff --git a/sys/arch/amd64/amd64/cpu.c b/sys/arch/amd64/amd64/cpu.c
index 542d961..60220bf 100644 (file)
--- a/sys/arch/amd64/amd64/cpu.c
+++ b/sys/arch/amd64/amd64/cpu.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: cpu.c,v 1.174 2023/07/28 06:36:16 guenther Exp $      */
+/*     $OpenBSD: cpu.c,v 1.175 2023/07/31 04:01:07 guenther Exp $      */
 /* $NetBSD: cpu.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $ */
 
 /*-
@@ -187,11 +187,7 @@ replacemeltdown(void)
 {
        static int replacedone = 0;
        struct cpu_info *ci = &cpu_info_primary;
-       int swapgs_vuln = 0, s;
-
-       if (replacedone)
-               return;
-       replacedone = 1;
+       int swapgs_vuln = 0, ibrs = 0, s;
 
        if (strcmp(cpu_vendor, "GenuineIntel") == 0) {
                int family = ci->ci_family;
@@ -208,9 +204,39 @@ replacemeltdown(void)
                        /* KnightsLanding */
                        swapgs_vuln = 0;
                }
+               if ((ci->ci_feature_sefflags_edx & SEFF0EDX_ARCH_CAP) &&
+                   (rdmsr(MSR_ARCH_CAPABILITIES) & ARCH_CAP_IBRS_ALL)) {
+                       ibrs = 2;
+               } else if (ci->ci_feature_sefflags_edx & SEFF0EDX_IBRS) {
+                       ibrs = 1;
+               }
+        } else if (strcmp(cpu_vendor, "AuthenticAMD") == 0 &&
+            ci->ci_pnfeatset >= 0x80000008) {
+               if (ci->ci_feature_amdspec_ebx & CPUIDEBX_IBRS_ALWAYSON) {
+                       ibrs = 2;
+               } else if ((ci->ci_feature_amdspec_ebx & CPUIDEBX_IBRS) &&
+                   (ci->ci_feature_amdspec_ebx & CPUIDEBX_IBRS_PREF)) {
+                       ibrs = 1;
+               }
        }
 
+       /* Enhanced IBRS: turn it on once on each CPU and don't touch again */
+       if (ibrs == 2)
+               wrmsr(MSR_SPEC_CTRL, SPEC_CTRL_IBRS);
+
+       if (replacedone)
+               return;
+       replacedone = 1;
+
        s = splhigh();
+       if (ibrs == 2 || (ci->ci_feature_sefflags_edx & SEFF0EDX_IBT)) {
+               extern const char _jmprax, _jmpr11, _jmpr13;
+               extern const short _jmprax_len, _jmpr11_len, _jmpr13_len;
+               codepatch_replace(CPTAG_RETPOLINE_RAX, &_jmprax, _jmprax_len);
+               codepatch_replace(CPTAG_RETPOLINE_R11, &_jmpr11, _jmpr11_len);
+               codepatch_replace(CPTAG_RETPOLINE_R13, &_jmpr13, _jmpr13_len);
+       }
+
        if (!cpu_meltdown)
                codepatch_nop(CPTAG_MELTDOWN_NOP);
        else {

diff --git a/sys/arch/amd64/amd64/locore.S b/sys/arch/amd64/amd64/locore.S
index 9024948..27938da 100644 (file)
--- a/sys/arch/amd64/amd64/locore.S
+++ b/sys/arch/amd64/amd64/locore.S
@@ -1,4 +1,4 @@
-/*     $OpenBSD: locore.S,v 1.139 2023/07/28 06:18:35 guenther Exp $   */
+/*     $OpenBSD: locore.S,v 1.140 2023/07/31 04:01:07 guenther Exp $   */
 /*     $NetBSD: locore.S,v 1.13 2004/03/25 18:33:17 drochner Exp $     */
 
 /*
@@ -474,7 +474,16 @@ bogus_proc_pmap:
 END(cpu_switchto)
 
 NENTRY(retpoline_rax)
+       CODEPATCH_START
        JMP_RETPOLINE(rax)
+       CODEPATCH_END(CPTAG_RETPOLINE_RAX)
+END(retpoline_rax)
+
+NENTRY(__x86_indirect_thunk_r11)
+       CODEPATCH_START
+       JMP_RETPOLINE(r11)
+       CODEPATCH_END(CPTAG_RETPOLINE_R11)
+END(__x86_indirect_thunk_r11)
 
 ENTRY(cpu_idle_cycle_hlt)
        RETGUARD_SETUP(cpu_idle_cycle_hlt, r11)

diff --git a/sys/arch/amd64/amd64/vector.S b/sys/arch/amd64/amd64/vector.S
index d3b31c7..af268a5 100644 (file)
--- a/sys/arch/amd64/amd64/vector.S
+++ b/sys/arch/amd64/amd64/vector.S
@@ -1,4 +1,4 @@
-/*     $OpenBSD: vector.S,v 1.93 2023/07/27 00:30:07 guenther Exp $    */
+/*     $OpenBSD: vector.S,v 1.94 2023/07/31 04:01:07 guenther Exp $    */
 /*     $NetBSD: vector.S,v 1.5 2004/06/28 09:13:11 fvdl Exp $  */
 
 /*
@@ -1269,7 +1269,9 @@ END(ioapic_level_stubs)
  * Soft interrupt handlers
  */
 NENTRY(retpoline_r13)
+       CODEPATCH_START
        JMP_RETPOLINE(r13)
+       CODEPATCH_END(CPTAG_RETPOLINE_R13)
 END(retpoline_r13)
 
 KIDTVEC(softtty)
@@ -1280,7 +1282,9 @@ KIDTVEC(softtty)
        movl    $X86_SOFTINTR_SOFTTTY,%edi
        call    softintr_dispatch
        decl    CPUVAR(IDEPTH)
+       CODEPATCH_START
        jmp     retpoline_r13
+       CODEPATCH_END(CPTAG_RETPOLINE_R13)
 END(Xsofttty)
 
 KIDTVEC(softnet)
@@ -1291,7 +1295,9 @@ KIDTVEC(softnet)
        movl    $X86_SOFTINTR_SOFTNET,%edi
        call    softintr_dispatch
        decl    CPUVAR(IDEPTH)
+       CODEPATCH_START
        jmp     retpoline_r13
+       CODEPATCH_END(CPTAG_RETPOLINE_R13)
 END(Xsoftnet)
 
 KIDTVEC(softclock)
@@ -1302,5 +1308,7 @@ KIDTVEC(softclock)
        movl    $X86_SOFTINTR_SOFTCLOCK,%edi
        call    softintr_dispatch
        decl    CPUVAR(IDEPTH)
+       CODEPATCH_START
        jmp     retpoline_r13
+       CODEPATCH_END(CPTAG_RETPOLINE_R13)
 END(Xsoftclock)

diff --git a/sys/arch/amd64/conf/Makefile.amd64 b/sys/arch/amd64/conf/Makefile.amd64
index 7759b87..6f760b1 100644 (file)
--- a/sys/arch/amd64/conf/Makefile.amd64
+++ b/sys/arch/amd64/conf/Makefile.amd64
@@ -1,4 +1,4 @@
-#      $OpenBSD: Makefile.amd64,v 1.132 2023/04/21 13:24:20 bluhm Exp $
+#      $OpenBSD: Makefile.amd64,v 1.133 2023/07/31 04:01:07 guenther Exp $
 
 # For instructions on building kernels consult the config(8) and options(4)
 # manual pages.
@@ -72,7 +72,7 @@ COPTIMIZE=    -Oz
 CMACHFLAGS+=   -mno-retpoline -fcf-protection=none
 .endif
 .else
-CMACHFLAGS+=   -fcf-protection=branch
+CMACHFLAGS+=   -mretpoline-external-thunk -fcf-protection=branch
 .endif
 .if ${COMPILER_VERSION:Mclang}
 NO_INTEGR_AS=  -no-integrated-as

diff --git a/sys/arch/amd64/include/codepatch.h b/sys/arch/amd64/include/codepatch.h
index 6401dab..6f2bdbe 100644 (file)
--- a/sys/arch/amd64/include/codepatch.h
+++ b/sys/arch/amd64/include/codepatch.h
@@ -1,4 +1,4 @@
-/*      $OpenBSD: codepatch.h,v 1.17 2023/07/31 01:33:57 guenther Exp $    */
+/*      $OpenBSD: codepatch.h,v 1.18 2023/07/31 04:01:07 guenther Exp $    */
 /*
  * Copyright (c) 2014-2015 Stefan Fritsch <sf@sfritsch.de>
  *
@@ -66,6 +66,9 @@ void codepatch_disable(void);
 #define CPTAG_FENCE_SWAPGS_MIS_TAKEN   11
 #define CPTAG_FENCE_NO_SAFE_SMAP       12
 #define CPTAG_XRSTORS                  13
+#define CPTAG_RETPOLINE_RAX            14
+#define CPTAG_RETPOLINE_R11            15
+#define CPTAG_RETPOLINE_R13            16
 
 /*
  * stac/clac SMAP instructions have lfence like semantics.  Let's