To mitigate against spectre attacks, AMD processors without the

IBRS feature need an lfence instruction after every near ret.  Place
them after all functions in the kernel which are implemented in
assembler.  Change the retguard macro so that the end of the lfence
instruction is 16-byte aligned now.  This prevents that the ret
instruction is at the end of a 32-byte boundary.  The latter would
cause a performance impact on certain Intel processors which have
a microcode update to mitigate the jump conditional code erratum.
See software techniques for managing speculation on AMD processors
revision 9.17.20 mitigation G-5.
See Intel mitigations for jump conditional code erratum revision
1.0 november 2019 2.4 software guidance and optimization methods.
OK deraadt@ mortimer@

diff --git a/sys/arch/amd64/amd64/acpi_wakecode.S b/sys/arch/amd64/amd64/acpi_wakecode.S
index ccca99e..eaf8e88 100644 (file)
--- a/sys/arch/amd64/amd64/acpi_wakecode.S
+++ b/sys/arch/amd64/amd64/acpi_wakecode.S
@@ -1,4 +1,4 @@
-/* $OpenBSD: acpi_wakecode.S,v 1.46 2018/10/04 05:00:40 guenther Exp $ */
+/* $OpenBSD: acpi_wakecode.S,v 1.47 2021/09/04 22:15:33 bluhm Exp $ */
 /*
  * Copyright (c) 2001 Takanori Watanabe <takawata@jp.freebsd.org>
  * Copyright (c) 2001 Mitsuru IWASAKI <iwasaki@jp.freebsd.org>
@@ -491,6 +491,7 @@ NENTRY(hibernate_activate_resume_pt_machdep)
 
 1:     RETGUARD_CHECK(hibernate_activate_resume_pt_machdep, r11)
        ret
+       lfence
 
        /*
         * Switch to the private resume-time hibernate stack
@@ -505,12 +506,14 @@ NENTRY(hibernate_switch_stack_machdep)
        /* On our own stack from here onward */
        RETGUARD_CHECK(hibernate_switch_stack_machdep, r11)
        ret
+       lfence
 
 NENTRY(hibernate_flush)
        RETGUARD_SETUP(hibernate_flush, r11)
        invlpg  HIBERNATE_INFLATE_PAGE
        RETGUARD_CHECK(hibernate_flush, r11)
        ret
+       lfence
 #endif /* HIBERNATE */
 
        /*
@@ -806,3 +809,4 @@ NENTRY(acpi_savecpu)
        movl    $1, %eax
        RETGUARD_CHECK(acpi_savecpu, r11)
        ret
+       lfence

diff --git a/sys/arch/amd64/amd64/aes_intel.S b/sys/arch/amd64/amd64/aes_intel.S
index 0181287..6848513 100644 (file)
--- a/sys/arch/amd64/amd64/aes_intel.S
+++ b/sys/arch/amd64/amd64/aes_intel.S
@@ -1,4 +1,4 @@
-/*     $OpenBSD: aes_intel.S,v 1.13 2018/07/09 08:39:28 mortimer Exp $ */
+/*     $OpenBSD: aes_intel.S,v 1.14 2021/09/04 22:15:33 bluhm Exp $    */
 
 /*
  * Implement AES algorithm in Intel AES-NI instructions.
@@ -117,6 +117,7 @@ _key_expansion_256a:
        add     $0x10,%rcx
        RETGUARD_CHECK(_key_expansion_128, rax)
        ret
+       lfence
 
 _key_expansion_192a:
        RETGUARD_SETUP(_key_expansion_192a, rax)
@@ -142,6 +143,7 @@ _key_expansion_192a:
        add     $0x20,%rcx
        RETGUARD_CHECK(_key_expansion_192a, rax)
        ret
+       lfence
 
 _key_expansion_192b:
        RETGUARD_SETUP(_key_expansion_192b, rax)
@@ -162,6 +164,7 @@ _key_expansion_192b:
        add     $0x10,%rcx
        RETGUARD_CHECK(_key_expansion_192b, rax)
        ret
+       lfence
 
 _key_expansion_256b:
        RETGUARD_SETUP(_key_expansion_256b, rax)
@@ -175,6 +178,7 @@ _key_expansion_256b:
        add     $0x10,%rcx
        RETGUARD_CHECK(_key_expansion_256b, rax)
        ret
+       lfence
 
 /*
  * void aesni_set_key(struct aesni_session *ses, uint8_t *key, size_t len)
@@ -278,6 +282,7 @@ ENTRY(aesni_set_key)
        jb      4b
        RETGUARD_CHECK(aesni_set_key, r11)
        ret
+       lfence
 
 /*
  * void aesni_enc(struct aesni_session *ses, uint8_t *dst, uint8_t *src)
@@ -290,6 +295,7 @@ ENTRY(aesni_enc)
        movups  STATE,(OUTP)            # output
        RETGUARD_CHECK(aesni_enc, r11)
        ret
+       lfence
 
 /*
  * _aesni_enc1:                internal ABI
@@ -348,6 +354,7 @@ _aesni_enc1:
        aesenclast KEY,STATE
        RETGUARD_CHECK(_aesni_enc1, rax)
        ret
+       lfence
 
 /*
  * _aesni_enc4:        internal ABI
@@ -457,6 +464,7 @@ _aesni_enc4:
        aesenclast KEY,STATE4
        RETGUARD_CHECK(_aesni_enc4, rax)
        ret
+       lfence
 
 /*
  * void aesni_dec(struct aesni_session *ses, uint8_t *dst, uint8_t *src)
@@ -470,6 +478,7 @@ ENTRY(aesni_dec)
        movups  STATE,(OUTP)            # output
        RETGUARD_CHECK(aesni_dec, r11)
        ret
+       lfence
 
 /*
  * _aesni_dec1:                internal ABI
@@ -528,6 +537,7 @@ _aesni_dec1:
        aesdeclast KEY,STATE
        RETGUARD_CHECK(_aesni_dec1, rax)
        ret
+       lfence
 
 /*
  * _aesni_dec4:        internal ABI
@@ -637,6 +647,7 @@ _aesni_dec4:
        aesdeclast KEY,STATE4
        RETGUARD_CHECK(_aesni_dec4, rax)
        ret
+       lfence
 
 #if 0
 /*
@@ -683,6 +694,7 @@ ENTRY(aesni_ecb_enc)
 3:
        RETGUARD_CHECK(aesni_ecb_enc, r11)
        ret
+       lfence
 
 /*
  * void aesni_ecb_dec(struct aesni_session *ses, uint8_t *dst, uint8_t *src,
@@ -729,6 +741,7 @@ ENTRY(aesni_ecb_dec)
 3:
        RETGUARD_CHECK(aesni_ecb_dec, r11)
        ret
+       lfence
 #endif
 
 /*
@@ -756,6 +769,7 @@ ENTRY(aesni_cbc_enc)
 2:
        RETGUARD_CHECK(aesni_cbc_enc, r11)
        ret
+       lfence
 
 /*
  * void aesni_cbc_dec(struct aesni_session *ses, uint8_t *dst, uint8_t *src,
@@ -815,6 +829,7 @@ ENTRY(aesni_cbc_dec)
 4:
        RETGUARD_CHECK(aesni_cbc_dec, r11)
        ret
+       lfence
 
 /*
  * _aesni_inc_init:    internal ABI
@@ -839,6 +854,7 @@ _aesni_inc_init:
        movd    CTR,TCTR_LOW
        RETGUARD_CHECK(_aesni_inc_init, rax)
        ret
+       lfence
 
 /*
  * _aesni_inc:         internal ABI
@@ -868,6 +884,7 @@ _aesni_inc:
        pshufb  BSWAP_MASK,IV
        RETGUARD_CHECK(_aesni_inc, rax)
        ret
+       lfence
 
 /*
  * void aesni_ctr_enc(struct aesni_session *ses, uint8_t *dst, uint8_t *src,
@@ -932,6 +949,7 @@ ENTRY(aesni_ctr_enc)
        RETGUARD_POP(r11)
        RETGUARD_CHECK(aesni_ctr_enc, r11)
        ret
+       lfence
 
 _aesni_gmac_gfmul:
        RETGUARD_SETUP(_aesni_gmac_gfmul, rax)
@@ -1000,6 +1018,7 @@ _aesni_gmac_gfmul:
        pxor    %xmm3,%xmm6             # the result is in xmm6
        RETGUARD_CHECK(_aesni_gmac_gfmul, rax)
        ret
+       lfence
 
 /*
  * void aesni_gmac_update(GHASH_CTX *ghash, uint8_t *src, size_t len)
@@ -1033,6 +1052,7 @@ ENTRY(aesni_gmac_update)
 2:
        RETGUARD_CHECK(aesni_gmac_update, r11)
        ret
+       lfence
 
 /*
  * void aesni_gmac_final(struct aesni_sess *ses, uint8_t *tag,
@@ -1048,6 +1068,7 @@ ENTRY(aesni_gmac_final)
        movdqu  STATE,(OUTP)            # output
        RETGUARD_CHECK(aesni_gmac_final, r11)
        ret
+       lfence
 
 /*
  * void aesni_xts_enc(struct aesni_xts_ctx *xts, uint8_t *dst, uint8_t *src,
@@ -1080,6 +1101,7 @@ ENTRY(aesni_xts_enc)
        RETGUARD_POP(r11)
        RETGUARD_CHECK(aesni_xts_enc, r11)
        ret
+       lfence
 
 /*
  * void aesni_xts_dec(struct aesni_xts_ctx *xts, uint8_t *dst, uint8_t *src,
@@ -1113,6 +1135,7 @@ ENTRY(aesni_xts_dec)
        RETGUARD_POP(r11)
        RETGUARD_CHECK(aesni_xts_dec, r11)
        ret
+       lfence
 
 /*
  * Prepare tweak as E_k2(IV). IV is specified as LE representation of a
@@ -1135,6 +1158,7 @@ _aesni_xts_tweak:
        RETGUARD_POP(rax)
        RETGUARD_CHECK(_aesni_xts_tweak, rax)
        ret
+       lfence
 
 /*
  * Exponentiate AES XTS tweak (in %xmm3).
@@ -1160,3 +1184,4 @@ _aesni_xts_tweak_exp:
 2:
        RETGUARD_CHECK(_aesni_xts_tweak_exp, rax)
        ret
+       lfence

diff --git a/sys/arch/amd64/amd64/copy.S b/sys/arch/amd64/amd64/copy.S
index 2e8b4a3..73dcefa 100644 (file)
--- a/sys/arch/amd64/amd64/copy.S
+++ b/sys/arch/amd64/amd64/copy.S
@@ -1,4 +1,4 @@
-/*     $OpenBSD: copy.S,v 1.13 2019/04/02 03:35:08 mortimer Exp $      */
+/*     $OpenBSD: copy.S,v 1.14 2021/09/04 22:15:33 bluhm Exp $ */
 /*     $NetBSD: copy.S,v 1.1 2003/04/26 18:39:26 fvdl Exp $    */
 
 /*
@@ -79,6 +79,7 @@ ENTRY(kcopy)
        xorq    %rax,%rax
        RETGUARD_CHECK(kcopy, r10)
        ret
+       lfence
 
 1:     addq    %rcx,%rdi               # copy backward
        addq    %rcx,%rsi
@@ -101,6 +102,7 @@ ENTRY(kcopy)
        xorq    %rax,%rax
        RETGUARD_CHECK(kcopy, r10)
        ret
+       lfence
 
 ENTRY(copyout)
        RETGUARD_SETUP(kcopy, r10)
@@ -138,6 +140,7 @@ ENTRY(copyout)
        xorl    %eax,%eax
        RETGUARD_CHECK(kcopy, r10)
        ret
+       lfence
 
 ENTRY(copyin)
        RETGUARD_SETUP(kcopy, r10)
@@ -177,6 +180,7 @@ ENTRY(copyin)
        xorl    %eax,%eax
        RETGUARD_CHECK(kcopy, r10)
        ret
+       lfence
 
 NENTRY(copy_fault)
        SMAP_CLAC
@@ -185,6 +189,7 @@ NENTRY(copy_fault)
        movl    $EFAULT,%eax
        RETGUARD_CHECK(kcopy, r10)
        ret
+       lfence
 
 ENTRY(copyoutstr)
        RETGUARD_SETUP(copyoutstr, r10)
@@ -285,6 +290,7 @@ copystr_return:
 8:
        RETGUARD_CHECK(copyoutstr, r10)
        ret
+       lfence
 
 ENTRY(copystr)
        RETGUARD_SETUP(copystr, r10)
@@ -316,6 +322,7 @@ ENTRY(copystr)
 7:
        RETGUARD_CHECK(copystr, r10)
        ret
+       lfence
 
        .section .rodata
        .globl _C_LABEL(_stac)

diff --git a/sys/arch/amd64/amd64/locore.S b/sys/arch/amd64/amd64/locore.S
index ec15e95..6e2aece 100644 (file)
--- a/sys/arch/amd64/amd64/locore.S
+++ b/sys/arch/amd64/amd64/locore.S
@@ -1,4 +1,4 @@
-/*     $OpenBSD: locore.S,v 1.125 2021/06/18 06:17:28 guenther Exp $   */
+/*     $OpenBSD: locore.S,v 1.126 2021/09/04 22:15:33 bluhm Exp $      */
 /*     $NetBSD: locore.S,v 1.13 2004/03/25 18:33:17 drochner Exp $     */
 
 /*
@@ -292,6 +292,7 @@ ENTRY(setjmp)
        xorl    %eax,%eax
        RETGUARD_CHECK(setjmp, r11)
        ret
+       lfence
 END(setjmp)
 
 ENTRY(longjmp)
@@ -310,6 +311,7 @@ ENTRY(longjmp)
        incl    %eax
        RETGUARD_CHECK(longjmp, r11)
        ret
+       lfence
 END(longjmp)
 #endif /* DDB */
 
@@ -508,6 +510,7 @@ switch_restored:
        popq    %rbx
        RETGUARD_CHECK(cpu_switchto, r11)
        ret
+       lfence
 
 #ifdef DIAGNOSTIC
        .globl  _C_LABEL(panic)
@@ -524,10 +527,12 @@ END(cpu_switchto)
 
 ENTRY(cpu_idle_enter)
        ret
+       lfence
 END(cpu_idle_enter)
 
 ENTRY(cpu_idle_leave)
        ret
+       lfence
 END(cpu_idle_leave)
 
 /* placed here for correct static branch prediction in cpu_idle_* */
@@ -543,6 +548,7 @@ ENTRY(cpu_idle_cycle)
        hlt
        RETGUARD_CHECK(cpu_idle_cycle, r11)
        ret
+       lfence
 END(cpu_idle_cycle)
 
 /*
@@ -556,6 +562,7 @@ ENTRY(savectx)
        movq    %rbp,PCB_RBP(%rdi)
        RETGUARD_CHECK(savectx, r11)
        ret
+       lfence
 END(savectx)
 
 IDTVEC(syscall32)
@@ -1055,10 +1062,12 @@ xrstor_fault:
        xorl    %eax, %eax
        RETGUARD_CHECK(xrstor_user, r11)
        ret
+       lfence
 NENTRY(xrstor_resume)
        movl    $1, %eax
        RETGUARD_CHECK(xrstor_user, r11)
        ret
+       lfence
 END(xrstor_user)
 
 ENTRY(fpusave)
@@ -1071,6 +1080,7 @@ ENTRY(fpusave)
        CODEPATCH_END(CPTAG_XSAVE)
        RETGUARD_CHECK(fpusave, r11)
        ret
+       lfence
 END(fpusave)
 
 ENTRY(fpusavereset)
@@ -1090,6 +1100,7 @@ ENTRY(fpusavereset)
        CODEPATCH_END(CPTAG_XRSTOR)
        RETGUARD_CHECK(fpusavereset, r11)
        ret
+       lfence
 END(fpusavereset)
 
 ENTRY(xsetbv_user)
@@ -1104,10 +1115,12 @@ xsetbv_fault:
        xorl    %eax, %eax
        RETGUARD_CHECK(xsetbv_user, r11)
        ret
+       lfence
 NENTRY(xsetbv_resume)
        movl    $1, %eax
        RETGUARD_CHECK(xsetbv_user, r11)
        ret
+       lfence
 END(xsetbv_user)
 
        .section .rodata
@@ -1142,6 +1155,7 @@ ENTRY(pagezero)
        sfence
        RETGUARD_CHECK(pagezero, r11)
        ret
+       lfence
 END(pagezero)
 
 /* int rdmsr_safe(u_int msr, uint64_t *data) */
@@ -1160,11 +1174,13 @@ rdmsr_safe_fault:
 
        RETGUARD_CHECK(rdmsr_safe, r10)
        ret
+       lfence
 
 NENTRY(rdmsr_resume)
        movl    $0x1, %eax
        RETGUARD_CHECK(rdmsr_safe, r10)
        ret
+       lfence
 END(rdmsr_safe)
 
 #if NXEN > 0

diff --git a/sys/arch/amd64/amd64/mds.S b/sys/arch/amd64/amd64/mds.S
index 27a0e6e..a57f863 100644 (file)
--- a/sys/arch/amd64/amd64/mds.S
+++ b/sys/arch/amd64/amd64/mds.S
@@ -1,4 +1,4 @@
-/*     $OpenBSD: mds.S,v 1.2 2019/11/29 17:47:10 mortimer Exp $        */
+/*     $OpenBSD: mds.S,v 1.3 2021/09/04 22:15:33 bluhm Exp $   */
 /*
  * Copyright (c) 2019 Philip Guenther <guenther@openbsd.org>
  *
@@ -47,6 +47,7 @@ ENTRY(mds_handler_ivb)
        movdqa  CPUVAR(MDS_TMP),%xmm0
        RETGUARD_CHECK(mds_handler_ivb, r11)
        retq
+       lfence
 END(mds_handler_ivb)
 
 ENTRY(mds_handler_bdw)
@@ -70,6 +71,7 @@ ENTRY(mds_handler_bdw)
        movdqa  CPUVAR(MDS_TMP),%xmm0
        RETGUARD_CHECK(mds_handler_bdw, r11)
        retq
+       lfence
 END(mds_handler_bdw)
 
 ENTRY(mds_handler_skl)
@@ -104,6 +106,7 @@ ENTRY(mds_handler_skl_sse)
        movdqa  CPUVAR(MDS_TMP),%xmm0
        RETGUARD_CHECK(mds_handler_skl_sse, r11)
        retq
+       lfence
 END(mds_handler_skl_sse)
 
 ENTRY(mds_handler_skl_avx)
@@ -130,6 +133,7 @@ ENTRY(mds_handler_skl_avx)
        vmovdqa CPUVAR(MDS_TMP),%ymm0
        RETGUARD_CHECK(mds_handler_skl_avx, r11)
        retq
+       lfence
 END(mds_handler_skl_avx)
 
 /* we don't support AVX512 yet */
@@ -158,6 +162,7 @@ ENTRY(mds_handler_skl_avx512)
        vmovdqa64       CPUVAR(MDS_TMP),%zmm0
        RETGUARD_CHECK(mds_handler_skl_avx512, r11)
        retq
+       lfence
 END(mds_handler_skl_avx512)
 #endif
 
@@ -177,6 +182,7 @@ ENTRY(mds_handler_silvermont)
        movdqa  CPUVAR(MDS_TMP),%xmm0
        RETGUARD_CHECK(mds_handler_silvermont, r11)
        retq
+       lfence
 END(mds_handler_silvermont)
 
 ENTRY(mds_handler_knights)
@@ -190,4 +196,5 @@ ENTRY(mds_handler_knights)
        mfence
        RETGUARD_CHECK(mds_handler_knights, r11)
        retq
+       lfence
 END(mds_handler_knights)

diff --git a/sys/arch/amd64/amd64/spl.S b/sys/arch/amd64/amd64/spl.S
index c478255..84c6c6f 100644 (file)
--- a/sys/arch/amd64/amd64/spl.S
+++ b/sys/arch/amd64/amd64/spl.S
@@ -1,4 +1,4 @@
-/*     $OpenBSD: spl.S,v 1.17 2018/07/10 16:01:26 deraadt Exp $        */
+/*     $OpenBSD: spl.S,v 1.18 2021/09/04 22:15:33 bluhm Exp $  */
 /*     $NetBSD: spl.S,v 1.3 2004/06/28 09:13:11 fvdl Exp $     */
 
 /*
@@ -89,6 +89,7 @@ _C_LABEL(splhigh):
        xchgl   %eax,CPUVAR(ILEVEL)
        RETGUARD_CHECK(splhigh, r11)
        ret
+       lfence
 
        .align 16, 0xcc
 _C_LABEL(splx):
@@ -99,6 +100,7 @@ _C_LABEL(splx):
        jnz     _C_LABEL(Xspllower)
        RETGUARD_CHECK(splx, r11)
        ret
+       lfence
 #endif /* PROF || GPROF */
 #endif
 
@@ -136,6 +138,7 @@ KIDTVEC(spllower)
        popq    %rbx
        RETGUARD_CHECK(Xspllower, r11)
        ret
+       lfence
 2:     bsrq    %rax,%rax
        btrq    %rax,CPUVAR(IPENDING)
        movq    CPUVAR(ISOURCES)(,%rax,8),%rax

diff --git a/sys/arch/amd64/amd64/vector.S b/sys/arch/amd64/amd64/vector.S
index cc8625b..825a661 100644 (file)
--- a/sys/arch/amd64/amd64/vector.S
+++ b/sys/arch/amd64/amd64/vector.S
@@ -1,4 +1,4 @@
-/*     $OpenBSD: vector.S,v 1.85 2021/09/03 16:45:44 jasper Exp $      */
+/*     $OpenBSD: vector.S,v 1.86 2021/09/04 22:15:33 bluhm Exp $       */
 /*     $NetBSD: vector.S,v 1.5 2004/06/28 09:13:11 fvdl Exp $  */
 
 /*
@@ -537,6 +537,7 @@ KUENTRY(x2apic_eoi)
        popq    %rcx
        popq    %rax
        ret
+       lfence
 END(x2apic_eoi)
 
 #if NLAPIC > 0

diff --git a/sys/arch/amd64/amd64/vmm_support.S b/sys/arch/amd64/amd64/vmm_support.S
index c328fb1..3be2f29 100644 (file)
--- a/sys/arch/amd64/amd64/vmm_support.S
+++ b/sys/arch/amd64/amd64/vmm_support.S
@@ -1,4 +1,4 @@
-/*     $OpenBSD: vmm_support.S,v 1.17 2021/02/13 07:47:37 mlarkin Exp $        */
+/*     $OpenBSD: vmm_support.S,v 1.18 2021/09/04 22:15:33 bluhm Exp $  */
 /*
  * Copyright (c) 2014 Mike Larkin <mlarkin@openbsd.org>
  *
@@ -58,6 +58,7 @@ _C_LABEL(vmm_dispatch_intr):
        cli
        callq   *%rdi
        ret
+       lfence
 
 _C_LABEL(vmxon):
        RETGUARD_SETUP(vmxon, r11)
@@ -67,10 +68,12 @@ _C_LABEL(vmxon):
        xorq    %rax, %rax
        RETGUARD_CHECK(vmxon, r11)
        ret
+       lfence
 failed_on:
        movq    $0x01, %rax
        RETGUARD_CHECK(vmxon, r11)
        ret
+       lfence
 
 _C_LABEL(vmxoff):
        RETGUARD_SETUP(vmxoff, r11)
@@ -80,10 +83,12 @@ _C_LABEL(vmxoff):
        xorq    %rax, %rax
        RETGUARD_CHECK(vmxoff, r11)
        ret
+       lfence
 failed_off:
        movq    $0x01, %rax
        RETGUARD_CHECK(vmxoff, r11)
        ret
+       lfence
 
 _C_LABEL(vmclear):
        RETGUARD_SETUP(vmclear, r11)
@@ -93,10 +98,12 @@ _C_LABEL(vmclear):
        xorq    %rax, %rax
        RETGUARD_CHECK(vmclear, r11)
        ret
+       lfence
 failed_clear:
        movq    $0x01, %rax
        RETGUARD_CHECK(vmclear, r11)
        ret
+       lfence
 
 _C_LABEL(vmptrld):
        RETGUARD_SETUP(vmptrld, r11)
@@ -106,10 +113,12 @@ _C_LABEL(vmptrld):
        xorq    %rax, %rax
        RETGUARD_CHECK(vmptrld, r11)
        ret
+       lfence
 failed_ptrld:
        movq    $0x01, %rax
        RETGUARD_CHECK(vmptrld, r11)
        ret
+       lfence
 
 _C_LABEL(vmptrst):
        RETGUARD_SETUP(vmptrst, r11)
@@ -119,10 +128,12 @@ _C_LABEL(vmptrst):
        xorq    %rax, %rax
        RETGUARD_CHECK(vmptrst, r11)
        ret
+       lfence
 failed_ptrst:
        movq    $0x01, %rax
        RETGUARD_CHECK(vmptrst, r11)
        ret
+       lfence
 
 _C_LABEL(vmwrite):
        RETGUARD_SETUP(vmwrite, r11)
@@ -132,10 +143,12 @@ _C_LABEL(vmwrite):
        xorq    %rax, %rax
        RETGUARD_CHECK(vmwrite, r11)
        ret
+       lfence
 failed_write:
        movq    $0x01, %rax
        RETGUARD_CHECK(vmwrite, r11)
        ret
+       lfence
 
 _C_LABEL(vmread):
        RETGUARD_SETUP(vmread, r11)
@@ -145,22 +158,26 @@ _C_LABEL(vmread):
        xorq    %rax, %rax
        RETGUARD_CHECK(vmread, r11)
        ret
+       lfence
 failed_read:
        movq    $0x01, %rax
        RETGUARD_CHECK(vmread, r11)
        ret
+       lfence
 
 _C_LABEL(invvpid):
        RETGUARD_SETUP(invvpid, r11)
        invvpid (%rsi), %rdi
        RETGUARD_CHECK(invvpid, r11)
        ret
+       lfence
 
 _C_LABEL(invept):
        RETGUARD_SETUP(invept, r11)
        invept (%rsi), %rdi
        RETGUARD_CHECK(invept, r11)
        ret
+       lfence
 
 _C_LABEL(vmx_enter_guest):
        RETGUARD_SETUP(vmx_enter_guest, r11)
@@ -534,6 +551,7 @@ restore_host:
        movq    %rdi, %rax
        RETGUARD_CHECK(vmx_enter_guest, r11)
        ret
+       lfence
 
 _C_LABEL(svm_enter_guest):
        RETGUARD_SETUP(svm_enter_guest, r11)
@@ -761,3 +779,4 @@ restore_host_svm:
 
        RETGUARD_CHECK(svm_enter_guest, r11)
        ret
+       lfence

diff --git a/sys/arch/amd64/include/asm.h b/sys/arch/amd64/include/asm.h
index fc57554..e642072 100644 (file)
--- a/sys/arch/amd64/include/asm.h
+++ b/sys/arch/amd64/include/asm.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: asm.h,v 1.19 2021/09/01 09:50:21 bluhm Exp $  */
+/*     $OpenBSD: asm.h,v 1.20 2021/09/04 22:15:33 bluhm Exp $  */
 /*     $NetBSD: asm.h,v 1.2 2003/05/02 18:05:47 yamt Exp $     */
 
 /*-
@@ -135,7 +135,7 @@
        cmpq (__retguard_ ## x)(%rip), %reg; \
        je 66f; \
        int3; int3; \
-       .zero (0xf - ((. - x) & 0xf)), 0xcc; \
+       .zero (0xf - ((. + 3 - x) & 0xf)), 0xcc; \
 66:
 # define RETGUARD_PUSH(reg) \
        pushq %reg

diff --git a/sys/arch/amd64/include/profile.h b/sys/arch/amd64/include/profile.h
index 9cb638f..7f63b92 100644 (file)
--- a/sys/arch/amd64/include/profile.h
+++ b/sys/arch/amd64/include/profile.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: profile.h,v 1.4 2012/08/22 17:19:35 pascal Exp $      */
+/*     $OpenBSD: profile.h,v 1.5 2021/09/04 22:15:33 bluhm Exp $       */
 /*     $NetBSD: profile.h,v 1.3 2003/11/28 23:22:45 fvdl Exp $ */
 
 /*
@@ -67,6 +67,7 @@ __asm(" .globl __mcount               \n"                     \
 "      movq    48(%rsp),%rax   \n"                     \
 "      leave                   \n"                     \
 "      ret                     \n"                     \
+"      lfence                  \n"                     \
 "      .size __mcount,.-__mcount");