From 6cbac32f9ea2203a6cfb69c3648aeb36e7aff937 Mon Sep 17 00:00:00 2001 From: guenther Date: Mon, 12 Feb 2024 01:18:17 +0000 Subject: [PATCH] Retpolines are an anti-pattern for IBT, so we need to shift protecting userspace from cross-process BTI to the kernel. Have each CPU track the last pmap run on in userspace and the last vmm VCPU in guest-mode and use the IBPB msr to flush predictors right before running in userspace on a different pmap or entering guest-mode on a different VCPU. Codepatch-nop the userspace bits and conditionalize the vmm bits to keep working if IBPB isn't supported. ok deraadt@ kettenis@ --- sys/arch/amd64/amd64/cpu.c | 7 ++++- sys/arch/amd64/amd64/genassym.cf | 3 +- sys/arch/amd64/amd64/locore.S | 44 ++++++++++++++++++++++++++---- sys/arch/amd64/amd64/vector.S | 9 +++++- sys/arch/amd64/amd64/vmm_machdep.c | 22 ++++++++++++++- sys/arch/amd64/include/codepatch.h | 3 +- sys/arch/amd64/include/cpu.h | 7 +++-- 7 files changed, 82 insertions(+), 13 deletions(-) diff --git a/sys/arch/amd64/amd64/cpu.c b/sys/arch/amd64/amd64/cpu.c index 42c907d908a..c89099a0f25 100644 --- a/sys/arch/amd64/amd64/cpu.c +++ b/sys/arch/amd64/amd64/cpu.c @@ -1,4 +1,4 @@ -/* $OpenBSD: cpu.c,v 1.178 2024/02/03 16:21:22 deraadt Exp $ */ +/* $OpenBSD: cpu.c,v 1.179 2024/02/12 01:18:17 guenther Exp $ */ /* $NetBSD: cpu.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $ */ /*- @@ -230,6 +230,11 @@ replacemeltdown(void) replacedone = 1; s = splhigh(); + + /* If we don't have IBRS/IBPB, then don't use IBPB */ + if ((ci->ci_feature_sefflags_edx & SEFF0EDX_IBRS) == 0) + codepatch_nop(CPTAG_IBPB_NOP); + if (ibrs == 2 || (ci->ci_feature_sefflags_edx & SEFF0EDX_IBT)) { extern const char _jmprax, _jmpr11, _jmpr13; extern const short _jmprax_len, _jmpr11_len, _jmpr13_len; diff --git a/sys/arch/amd64/amd64/genassym.cf b/sys/arch/amd64/amd64/genassym.cf index dda8a9d36cf..8ad27fbc5b1 100644 --- a/sys/arch/amd64/amd64/genassym.cf +++ b/sys/arch/amd64/amd64/genassym.cf @@ -1,4 +1,4 @@ -# $OpenBSD: genassym.cf,v 1.44 2023/01/16 00:05:18 deraadt Exp $ +# $OpenBSD: genassym.cf,v 1.45 2024/02/12 01:18:17 guenther Exp $ # Written by Artur Grabowski art@openbsd.org, Public Domain include @@ -108,6 +108,7 @@ member CPU_INFO_APICID ci_apicid member CPU_INFO_RESCHED ci_want_resched member CPU_INFO_CURPROC ci_curproc member CPU_INFO_PROC_PMAP ci_proc_pmap +member CPU_INFO_USER_PMAP ci_user_pmap member CPU_INFO_CURPCB ci_curpcb member CPU_INFO_IDLE_PCB ci_idle_pcb member CPU_INFO_ILEVEL ci_ilevel diff --git a/sys/arch/amd64/amd64/locore.S b/sys/arch/amd64/amd64/locore.S index b8b9c3274d8..1365cf9be5a 100644 --- a/sys/arch/amd64/amd64/locore.S +++ b/sys/arch/amd64/amd64/locore.S @@ -1,4 +1,4 @@ -/* $OpenBSD: locore.S,v 1.144 2023/12/12 15:30:55 deraadt Exp $ */ +/* $OpenBSD: locore.S,v 1.145 2024/02/12 01:18:17 guenther Exp $ */ /* $NetBSD: locore.S,v 1.13 2004/03/25 18:33:17 drochner Exp $ */ /* @@ -595,6 +595,22 @@ GENTRY(Xsyscall) jz .Lsyscall_restore_fsbase .Lsyscall_restore_registers: + /* + * If the pmap we're now on isn't the same as the one we + * were on last time we were in userspace, then use IBPB + * to prevent cross-process branch-target injection. + */ + CODEPATCH_START + movq CPUVAR(PROC_PMAP),%rbx + cmpq CPUVAR(USER_PMAP),%rbx + je 1f + xorl %edx,%edx + movl $PRED_CMD_IBPB,%eax + movl $MSR_PRED_CMD,%ecx + wrmsr + movq %rbx,CPUVAR(USER_PMAP) +1: + CODEPATCH_END(CPTAG_IBPB_NOP) call pku_xonly RET_STACK_REFILL_WITH_RCX @@ -758,17 +774,33 @@ intr_user_exit_post_ast: testl $CPUPF_USERXSTATE,CPUVAR(PFLAGS) jz .Lintr_restore_xstate + /* Restore FS.base if it's not already in the CPU */ + testl $CPUPF_USERSEGS,CPUVAR(PFLAGS) + jz .Lintr_restore_fsbase + +.Lintr_restore_registers: #ifdef DIAGNOSTIC /* no more C calls after this, so check the SPL */ cmpl $0,CPUVAR(ILEVEL) jne .Luser_spl_not_lowered #endif /* DIAGNOSTIC */ - /* Restore FS.base if it's not already in the CPU */ - testl $CPUPF_USERSEGS,CPUVAR(PFLAGS) - jz .Lintr_restore_fsbase - -.Lintr_restore_registers: + /* + * If the pmap we're now on isn't the same as the one we + * were on last time we were in userspace, then use IBPB + * to prevent cross-process branch-target injection. + */ + CODEPATCH_START + movq CPUVAR(PROC_PMAP),%rbx + cmpq CPUVAR(USER_PMAP),%rbx + je 1f + xorl %edx,%edx + movl $PRED_CMD_IBPB,%eax + movl $MSR_PRED_CMD,%ecx + wrmsr + movq %rbx,CPUVAR(USER_PMAP) +1: + CODEPATCH_END(CPTAG_IBPB_NOP) call pku_xonly RET_STACK_REFILL_WITH_RCX diff --git a/sys/arch/amd64/amd64/vector.S b/sys/arch/amd64/amd64/vector.S index af268a5295e..de24a291fd7 100644 --- a/sys/arch/amd64/amd64/vector.S +++ b/sys/arch/amd64/amd64/vector.S @@ -1,4 +1,4 @@ -/* $OpenBSD: vector.S,v 1.94 2023/07/31 04:01:07 guenther Exp $ */ +/* $OpenBSD: vector.S,v 1.95 2024/02/12 01:18:17 guenther Exp $ */ /* $NetBSD: vector.S,v 1.5 2004/06/28 09:13:11 fvdl Exp $ */ /* @@ -149,6 +149,13 @@ INTRENTRY_LABEL(calltrap_specstk): movq %r12,%rax movq %r13,%rdx wrmsr + /* who knows what happened in this trap; use IPBP on the way out */ + CODEPATCH_START + xorl %edx,%edx + movl $PRED_CMD_IBPB,%eax + movl $MSR_PRED_CMD,%ecx + wrmsr + CODEPATCH_END(CPTAG_IBPB_NOP) call pku_xonly popq %rdi popq %rsi diff --git a/sys/arch/amd64/amd64/vmm_machdep.c b/sys/arch/amd64/amd64/vmm_machdep.c index 0bbd2a407bb..ccc3c809557 100644 --- a/sys/arch/amd64/amd64/vmm_machdep.c +++ b/sys/arch/amd64/amd64/vmm_machdep.c @@ -1,4 +1,4 @@ -/* $OpenBSD: vmm_machdep.c,v 1.16 2024/01/31 05:49:33 guenther Exp $ */ +/* $OpenBSD: vmm_machdep.c,v 1.17 2024/02/12 01:18:17 guenther Exp $ */ /* * Copyright (c) 2014 Mike Larkin * @@ -4185,6 +4185,16 @@ vcpu_run_vmx(struct vcpu *vcpu, struct vm_run_params *vrp) TRACEPOINT(vmm, guest_enter, vcpu, vrp); + /* + * If we're resuming to a different VCPU and have IBPB, + * then use it to prevent cross-VM branch-target injection. + */ + if (ci->ci_guest_vcpu != vcpu && + (ci->ci_feature_sefflags_edx & SEFF0EDX_IBRS)) { + wrmsr(MSR_PRED_CMD, PRED_CMD_IBPB); + ci->ci_guest_vcpu = vcpu; + } + /* Restore any guest PKRU state. */ if (vmm_softc->sc_md.pkru_enabled) wrpkru(0, vcpu->vc_pkru); @@ -6498,6 +6508,16 @@ vcpu_run_svm(struct vcpu *vcpu, struct vm_run_params *vrp) break; } + /* + * If we're resuming to a different VCPU and have IBPB, + * then use it to prevent cross-VM branch-target injection. + */ + if (ci->ci_guest_vcpu != vcpu && + (ci->ci_feature_sefflags_edx & SEFF0EDX_IBRS)) { + wrmsr(MSR_PRED_CMD, PRED_CMD_IBPB); + ci->ci_guest_vcpu = vcpu; + } + /* Restore any guest PKRU state. */ if (vmm_softc->sc_md.pkru_enabled) wrpkru(0, vcpu->vc_pkru); diff --git a/sys/arch/amd64/include/codepatch.h b/sys/arch/amd64/include/codepatch.h index 6f2bdbe184e..2ccb638a8e8 100644 --- a/sys/arch/amd64/include/codepatch.h +++ b/sys/arch/amd64/include/codepatch.h @@ -1,4 +1,4 @@ -/* $OpenBSD: codepatch.h,v 1.18 2023/07/31 04:01:07 guenther Exp $ */ +/* $OpenBSD: codepatch.h,v 1.19 2024/02/12 01:18:17 guenther Exp $ */ /* * Copyright (c) 2014-2015 Stefan Fritsch * @@ -69,6 +69,7 @@ void codepatch_disable(void); #define CPTAG_RETPOLINE_RAX 14 #define CPTAG_RETPOLINE_R11 15 #define CPTAG_RETPOLINE_R13 16 +#define CPTAG_IBPB_NOP 17 /* * stac/clac SMAP instructions have lfence like semantics. Let's diff --git a/sys/arch/amd64/include/cpu.h b/sys/arch/amd64/include/cpu.h index 5c209b90aa2..dd0537cb164 100644 --- a/sys/arch/amd64/include/cpu.h +++ b/sys/arch/amd64/include/cpu.h @@ -1,4 +1,4 @@ -/* $OpenBSD: cpu.h,v 1.161 2024/02/03 16:21:22 deraadt Exp $ */ +/* $OpenBSD: cpu.h,v 1.162 2024/02/12 01:18:17 guenther Exp $ */ /* $NetBSD: cpu.h,v 1.1 2003/04/26 18:39:39 fvdl Exp $ */ /*- @@ -98,6 +98,7 @@ union vmm_cpu_cap { * o owned (read/modified only) by this CPU */ struct x86_64_tss; +struct vcpu; struct cpu_info { /* * The beginning of this structure in mapped in the userspace "u-k" @@ -130,7 +131,8 @@ struct cpu_info { struct proc *ci_curproc; /* [o] */ struct schedstate_percpu ci_schedstate; /* scheduler state */ - struct pmap *ci_proc_pmap; /* last userspace pmap */ + struct pmap *ci_proc_pmap; /* active, non-kernel pmap */ + struct pmap *ci_user_pmap; /* [o] last pmap used in userspace */ struct pcb *ci_curpcb; /* [o] */ struct pcb *ci_idle_pcb; /* [o] */ @@ -219,6 +221,7 @@ struct cpu_info { union vmm_cpu_cap ci_vmm_cap; paddr_t ci_vmxon_region_pa; struct vmxon_region *ci_vmxon_region; + struct vcpu *ci_guest_vcpu; /* [o] last vcpu resumed */ char ci_panicbuf[512]; -- 2.20.1