From: bluhm <bluhm@openbsd.org>
Date: Sat, 4 Sep 2021 22:15:33 +0000 (+0000)
Subject: To mitigate against spectre attacks, AMD processors without the
X-Git-Url: http://artulab.com/gitweb/?a=commitdiff_plain;h=3dd0809fbcf33671ef9b3771096948131e7cafab;p=openbsd

To mitigate against spectre attacks, AMD processors without the
IBRS feature need an lfence instruction after every near ret.  Place
them after all functions in the kernel which are implemented in
assembler.  Change the retguard macro so that the end of the lfence
instruction is 16-byte aligned now.  This prevents that the ret
instruction is at the end of a 32-byte boundary.  The latter would
cause a performance impact on certain Intel processors which have
a microcode update to mitigate the jump conditional code erratum.
See software techniques for managing speculation on AMD processors
revision 9.17.20 mitigation G-5.
See Intel mitigations for jump conditional code erratum revision
1.0 november 2019 2.4 software guidance and optimization methods.
OK deraadt@ mortimer@
---

diff --git a/sys/arch/amd64/amd64/acpi_wakecode.S b/sys/arch/amd64/amd64/acpi_wakecode.S
index ccca99ef73c..eaf8e887c83 100644
--- a/sys/arch/amd64/amd64/acpi_wakecode.S
+++ b/sys/arch/amd64/amd64/acpi_wakecode.S
@@ -1,4 +1,4 @@
-/* $OpenBSD: acpi_wakecode.S,v 1.46 2018/10/04 05:00:40 guenther Exp $ */
+/* $OpenBSD: acpi_wakecode.S,v 1.47 2021/09/04 22:15:33 bluhm Exp $ */
 /*
  * Copyright (c) 2001 Takanori Watanabe <takawata@jp.freebsd.org>
  * Copyright (c) 2001 Mitsuru IWASAKI <iwasaki@jp.freebsd.org>
@@ -491,6 +491,7 @@ NENTRY(hibernate_activate_resume_pt_machdep)
 
 1:	RETGUARD_CHECK(hibernate_activate_resume_pt_machdep, r11)
 	ret
+	lfence
 
 	/*
 	 * Switch to the private resume-time hibernate stack
@@ -505,12 +506,14 @@ NENTRY(hibernate_switch_stack_machdep)
 	/* On our own stack from here onward */
 	RETGUARD_CHECK(hibernate_switch_stack_machdep, r11)
 	ret
+	lfence
 
 NENTRY(hibernate_flush)
 	RETGUARD_SETUP(hibernate_flush, r11)
 	invlpg	HIBERNATE_INFLATE_PAGE
 	RETGUARD_CHECK(hibernate_flush, r11)
 	ret
+	lfence
 #endif /* HIBERNATE */
 
 	/*
@@ -806,3 +809,4 @@ NENTRY(acpi_savecpu)
 	movl	$1, %eax
 	RETGUARD_CHECK(acpi_savecpu, r11)
 	ret
+	lfence
diff --git a/sys/arch/amd64/amd64/aes_intel.S b/sys/arch/amd64/amd64/aes_intel.S
index 0181287cf52..6848513c20e 100644
--- a/sys/arch/amd64/amd64/aes_intel.S
+++ b/sys/arch/amd64/amd64/aes_intel.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: aes_intel.S,v 1.13 2018/07/09 08:39:28 mortimer Exp $	*/
+/*	$OpenBSD: aes_intel.S,v 1.14 2021/09/04 22:15:33 bluhm Exp $	*/
 
 /*
  * Implement AES algorithm in Intel AES-NI instructions.
@@ -117,6 +117,7 @@ _key_expansion_256a:
 	add	$0x10,%rcx
 	RETGUARD_CHECK(_key_expansion_128, rax)
 	ret
+	lfence
 
 _key_expansion_192a:
 	RETGUARD_SETUP(_key_expansion_192a, rax)
@@ -142,6 +143,7 @@ _key_expansion_192a:
 	add	$0x20,%rcx
 	RETGUARD_CHECK(_key_expansion_192a, rax)
 	ret
+	lfence
 
 _key_expansion_192b:
 	RETGUARD_SETUP(_key_expansion_192b, rax)
@@ -162,6 +164,7 @@ _key_expansion_192b:
 	add	$0x10,%rcx
 	RETGUARD_CHECK(_key_expansion_192b, rax)
 	ret
+	lfence
 
 _key_expansion_256b:
 	RETGUARD_SETUP(_key_expansion_256b, rax)
@@ -175,6 +178,7 @@ _key_expansion_256b:
 	add	$0x10,%rcx
 	RETGUARD_CHECK(_key_expansion_256b, rax)
 	ret
+	lfence
 
 /*
  * void aesni_set_key(struct aesni_session *ses, uint8_t *key, size_t len)
@@ -278,6 +282,7 @@ ENTRY(aesni_set_key)
 	jb	4b
 	RETGUARD_CHECK(aesni_set_key, r11)
 	ret
+	lfence
 
 /*
  * void aesni_enc(struct aesni_session *ses, uint8_t *dst, uint8_t *src)
@@ -290,6 +295,7 @@ ENTRY(aesni_enc)
 	movups	STATE,(OUTP)		# output
 	RETGUARD_CHECK(aesni_enc, r11)
 	ret
+	lfence
 
 /*
  * _aesni_enc1:		internal ABI
@@ -348,6 +354,7 @@ _aesni_enc1:
 	aesenclast KEY,STATE
 	RETGUARD_CHECK(_aesni_enc1, rax)
 	ret
+	lfence
 
 /*
  * _aesni_enc4:	internal ABI
@@ -457,6 +464,7 @@ _aesni_enc4:
 	aesenclast KEY,STATE4
 	RETGUARD_CHECK(_aesni_enc4, rax)
 	ret
+	lfence
 
 /*
  * void aesni_dec(struct aesni_session *ses, uint8_t *dst, uint8_t *src)
@@ -470,6 +478,7 @@ ENTRY(aesni_dec)
 	movups	STATE,(OUTP)		# output
 	RETGUARD_CHECK(aesni_dec, r11)
 	ret
+	lfence
 
 /*
  * _aesni_dec1:		internal ABI
@@ -528,6 +537,7 @@ _aesni_dec1:
 	aesdeclast KEY,STATE
 	RETGUARD_CHECK(_aesni_dec1, rax)
 	ret
+	lfence
 
 /*
  * _aesni_dec4:	internal ABI
@@ -637,6 +647,7 @@ _aesni_dec4:
 	aesdeclast KEY,STATE4
 	RETGUARD_CHECK(_aesni_dec4, rax)
 	ret
+	lfence
 
 #if 0
 /*
@@ -683,6 +694,7 @@ ENTRY(aesni_ecb_enc)
 3:
 	RETGUARD_CHECK(aesni_ecb_enc, r11)
 	ret
+	lfence
 
 /*
  * void aesni_ecb_dec(struct aesni_session *ses, uint8_t *dst, uint8_t *src,
@@ -729,6 +741,7 @@ ENTRY(aesni_ecb_dec)
 3:
 	RETGUARD_CHECK(aesni_ecb_dec, r11)
 	ret
+	lfence
 #endif
 
 /*
@@ -756,6 +769,7 @@ ENTRY(aesni_cbc_enc)
 2:
 	RETGUARD_CHECK(aesni_cbc_enc, r11)
 	ret
+	lfence
 
 /*
  * void aesni_cbc_dec(struct aesni_session *ses, uint8_t *dst, uint8_t *src,
@@ -815,6 +829,7 @@ ENTRY(aesni_cbc_dec)
 4:
 	RETGUARD_CHECK(aesni_cbc_dec, r11)
 	ret
+	lfence
 
 /*
  * _aesni_inc_init:	internal ABI
@@ -839,6 +854,7 @@ _aesni_inc_init:
 	movd	CTR,TCTR_LOW
 	RETGUARD_CHECK(_aesni_inc_init, rax)
 	ret
+	lfence
 
 /*
  * _aesni_inc:		internal ABI
@@ -868,6 +884,7 @@ _aesni_inc:
 	pshufb	BSWAP_MASK,IV
 	RETGUARD_CHECK(_aesni_inc, rax)
 	ret
+	lfence
 
 /*
  * void aesni_ctr_enc(struct aesni_session *ses, uint8_t *dst, uint8_t *src,
@@ -932,6 +949,7 @@ ENTRY(aesni_ctr_enc)
 	RETGUARD_POP(r11)
 	RETGUARD_CHECK(aesni_ctr_enc, r11)
 	ret
+	lfence
 
 _aesni_gmac_gfmul:
 	RETGUARD_SETUP(_aesni_gmac_gfmul, rax)
@@ -1000,6 +1018,7 @@ _aesni_gmac_gfmul:
 	pxor	%xmm3,%xmm6		# the result is in xmm6
 	RETGUARD_CHECK(_aesni_gmac_gfmul, rax)
 	ret
+	lfence
 
 /*
  * void aesni_gmac_update(GHASH_CTX *ghash, uint8_t *src, size_t len)
@@ -1033,6 +1052,7 @@ ENTRY(aesni_gmac_update)
 2:
 	RETGUARD_CHECK(aesni_gmac_update, r11)
 	ret
+	lfence
 
 /*
  * void aesni_gmac_final(struct aesni_sess *ses, uint8_t *tag,
@@ -1048,6 +1068,7 @@ ENTRY(aesni_gmac_final)
 	movdqu	STATE,(OUTP)		# output
 	RETGUARD_CHECK(aesni_gmac_final, r11)
 	ret
+	lfence
 
 /*
  * void aesni_xts_enc(struct aesni_xts_ctx *xts, uint8_t *dst, uint8_t *src,
@@ -1080,6 +1101,7 @@ ENTRY(aesni_xts_enc)
 	RETGUARD_POP(r11)
 	RETGUARD_CHECK(aesni_xts_enc, r11)
 	ret
+	lfence
 
 /*
  * void aesni_xts_dec(struct aesni_xts_ctx *xts, uint8_t *dst, uint8_t *src,
@@ -1113,6 +1135,7 @@ ENTRY(aesni_xts_dec)
 	RETGUARD_POP(r11)
 	RETGUARD_CHECK(aesni_xts_dec, r11)
 	ret
+	lfence
 
 /*
  * Prepare tweak as E_k2(IV). IV is specified as LE representation of a
@@ -1135,6 +1158,7 @@ _aesni_xts_tweak:
 	RETGUARD_POP(rax)
 	RETGUARD_CHECK(_aesni_xts_tweak, rax)
 	ret
+	lfence
 
 /*
  * Exponentiate AES XTS tweak (in %xmm3).
@@ -1160,3 +1184,4 @@ _aesni_xts_tweak_exp:
 2:
 	RETGUARD_CHECK(_aesni_xts_tweak_exp, rax)
 	ret
+	lfence
diff --git a/sys/arch/amd64/amd64/copy.S b/sys/arch/amd64/amd64/copy.S
index 2e8b4a328c7..73dcefa058f 100644
--- a/sys/arch/amd64/amd64/copy.S
+++ b/sys/arch/amd64/amd64/copy.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: copy.S,v 1.13 2019/04/02 03:35:08 mortimer Exp $	*/
+/*	$OpenBSD: copy.S,v 1.14 2021/09/04 22:15:33 bluhm Exp $	*/
 /*	$NetBSD: copy.S,v 1.1 2003/04/26 18:39:26 fvdl Exp $	*/
 
 /*
@@ -79,6 +79,7 @@ ENTRY(kcopy)
 	xorq	%rax,%rax
 	RETGUARD_CHECK(kcopy, r10)
 	ret
+	lfence
 
 1:	addq	%rcx,%rdi		# copy backward
 	addq	%rcx,%rsi
@@ -101,6 +102,7 @@ ENTRY(kcopy)
 	xorq	%rax,%rax
 	RETGUARD_CHECK(kcopy, r10)
 	ret
+	lfence
 
 ENTRY(copyout)
 	RETGUARD_SETUP(kcopy, r10)
@@ -138,6 +140,7 @@ ENTRY(copyout)
 	xorl	%eax,%eax
 	RETGUARD_CHECK(kcopy, r10)
 	ret
+	lfence
 
 ENTRY(copyin)
 	RETGUARD_SETUP(kcopy, r10)
@@ -177,6 +180,7 @@ ENTRY(copyin)
 	xorl	%eax,%eax
 	RETGUARD_CHECK(kcopy, r10)
 	ret
+	lfence
 
 NENTRY(copy_fault)
 	SMAP_CLAC
@@ -185,6 +189,7 @@ NENTRY(copy_fault)
 	movl	$EFAULT,%eax
 	RETGUARD_CHECK(kcopy, r10)
 	ret
+	lfence
 
 ENTRY(copyoutstr)
 	RETGUARD_SETUP(copyoutstr, r10)
@@ -285,6 +290,7 @@ copystr_return:
 8:
 	RETGUARD_CHECK(copyoutstr, r10)
 	ret
+	lfence
 
 ENTRY(copystr)
 	RETGUARD_SETUP(copystr, r10)
@@ -316,6 +322,7 @@ ENTRY(copystr)
 7:
 	RETGUARD_CHECK(copystr, r10)
 	ret
+	lfence
 
 	.section .rodata
 	.globl _C_LABEL(_stac)
diff --git a/sys/arch/amd64/amd64/locore.S b/sys/arch/amd64/amd64/locore.S
index ec15e95f90f..6e2aece5c5e 100644
--- a/sys/arch/amd64/amd64/locore.S
+++ b/sys/arch/amd64/amd64/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.125 2021/06/18 06:17:28 guenther Exp $	*/
+/*	$OpenBSD: locore.S,v 1.126 2021/09/04 22:15:33 bluhm Exp $	*/
 /*	$NetBSD: locore.S,v 1.13 2004/03/25 18:33:17 drochner Exp $	*/
 
 /*
@@ -292,6 +292,7 @@ ENTRY(setjmp)
 	xorl	%eax,%eax
 	RETGUARD_CHECK(setjmp, r11)
 	ret
+	lfence
 END(setjmp)
 
 ENTRY(longjmp)
@@ -310,6 +311,7 @@ ENTRY(longjmp)
 	incl	%eax
 	RETGUARD_CHECK(longjmp, r11)
 	ret
+	lfence
 END(longjmp)
 #endif /* DDB */
 
@@ -508,6 +510,7 @@ switch_restored:
 	popq	%rbx
 	RETGUARD_CHECK(cpu_switchto, r11)
 	ret
+	lfence
 
 #ifdef DIAGNOSTIC
 	.globl	_C_LABEL(panic)
@@ -524,10 +527,12 @@ END(cpu_switchto)
 
 ENTRY(cpu_idle_enter)
 	ret
+	lfence
 END(cpu_idle_enter)
 
 ENTRY(cpu_idle_leave)
 	ret
+	lfence
 END(cpu_idle_leave)
 
 /* placed here for correct static branch prediction in cpu_idle_* */
@@ -543,6 +548,7 @@ ENTRY(cpu_idle_cycle)
 	hlt
 	RETGUARD_CHECK(cpu_idle_cycle, r11)
 	ret
+	lfence
 END(cpu_idle_cycle)
 
 /*
@@ -556,6 +562,7 @@ ENTRY(savectx)
 	movq	%rbp,PCB_RBP(%rdi)
 	RETGUARD_CHECK(savectx, r11)
 	ret
+	lfence
 END(savectx)
 
 IDTVEC(syscall32)
@@ -1055,10 +1062,12 @@ xrstor_fault:
 	xorl	%eax, %eax
 	RETGUARD_CHECK(xrstor_user, r11)
 	ret
+	lfence
 NENTRY(xrstor_resume)
 	movl	$1, %eax
 	RETGUARD_CHECK(xrstor_user, r11)
 	ret
+	lfence
 END(xrstor_user)
 
 ENTRY(fpusave)
@@ -1071,6 +1080,7 @@ ENTRY(fpusave)
 	CODEPATCH_END(CPTAG_XSAVE)
 	RETGUARD_CHECK(fpusave, r11)
 	ret
+	lfence
 END(fpusave)
 
 ENTRY(fpusavereset)
@@ -1090,6 +1100,7 @@ ENTRY(fpusavereset)
 	CODEPATCH_END(CPTAG_XRSTOR)
 	RETGUARD_CHECK(fpusavereset, r11)
 	ret
+	lfence
 END(fpusavereset)
 
 ENTRY(xsetbv_user)
@@ -1104,10 +1115,12 @@ xsetbv_fault:
 	xorl	%eax, %eax
 	RETGUARD_CHECK(xsetbv_user, r11)
 	ret
+	lfence
 NENTRY(xsetbv_resume)
 	movl	$1, %eax
 	RETGUARD_CHECK(xsetbv_user, r11)
 	ret
+	lfence
 END(xsetbv_user)
 
 	.section .rodata
@@ -1142,6 +1155,7 @@ ENTRY(pagezero)
 	sfence
 	RETGUARD_CHECK(pagezero, r11)
 	ret
+	lfence
 END(pagezero)
 
 /* int rdmsr_safe(u_int msr, uint64_t *data) */
@@ -1160,11 +1174,13 @@ rdmsr_safe_fault:
 
 	RETGUARD_CHECK(rdmsr_safe, r10)
 	ret
+	lfence
 
 NENTRY(rdmsr_resume)
 	movl	$0x1, %eax
 	RETGUARD_CHECK(rdmsr_safe, r10)
 	ret
+	lfence
 END(rdmsr_safe)
 
 #if NXEN > 0
diff --git a/sys/arch/amd64/amd64/mds.S b/sys/arch/amd64/amd64/mds.S
index 27a0e6e93b0..a57f8634caa 100644
--- a/sys/arch/amd64/amd64/mds.S
+++ b/sys/arch/amd64/amd64/mds.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mds.S,v 1.2 2019/11/29 17:47:10 mortimer Exp $	*/
+/*	$OpenBSD: mds.S,v 1.3 2021/09/04 22:15:33 bluhm Exp $	*/
 /*
  * Copyright (c) 2019 Philip Guenther <guenther@openbsd.org>
  *
@@ -47,6 +47,7 @@ ENTRY(mds_handler_ivb)
 	movdqa	CPUVAR(MDS_TMP),%xmm0
 	RETGUARD_CHECK(mds_handler_ivb, r11)
 	retq
+	lfence
 END(mds_handler_ivb)
 
 ENTRY(mds_handler_bdw)
@@ -70,6 +71,7 @@ ENTRY(mds_handler_bdw)
 	movdqa	CPUVAR(MDS_TMP),%xmm0
 	RETGUARD_CHECK(mds_handler_bdw, r11)
 	retq
+	lfence
 END(mds_handler_bdw)
 
 ENTRY(mds_handler_skl)
@@ -104,6 +106,7 @@ ENTRY(mds_handler_skl_sse)
 	movdqa	CPUVAR(MDS_TMP),%xmm0
 	RETGUARD_CHECK(mds_handler_skl_sse, r11)
 	retq
+	lfence
 END(mds_handler_skl_sse)
 
 ENTRY(mds_handler_skl_avx)
@@ -130,6 +133,7 @@ ENTRY(mds_handler_skl_avx)
 	vmovdqa	CPUVAR(MDS_TMP),%ymm0
 	RETGUARD_CHECK(mds_handler_skl_avx, r11)
 	retq
+	lfence
 END(mds_handler_skl_avx)
 
 /* we don't support AVX512 yet */
@@ -158,6 +162,7 @@ ENTRY(mds_handler_skl_avx512)
 	vmovdqa64	CPUVAR(MDS_TMP),%zmm0
 	RETGUARD_CHECK(mds_handler_skl_avx512, r11)
 	retq
+	lfence
 END(mds_handler_skl_avx512)
 #endif
 
@@ -177,6 +182,7 @@ ENTRY(mds_handler_silvermont)
 	movdqa	CPUVAR(MDS_TMP),%xmm0
 	RETGUARD_CHECK(mds_handler_silvermont, r11)
 	retq
+	lfence
 END(mds_handler_silvermont)
 
 ENTRY(mds_handler_knights)
@@ -190,4 +196,5 @@ ENTRY(mds_handler_knights)
 	mfence
 	RETGUARD_CHECK(mds_handler_knights, r11)
 	retq
+	lfence
 END(mds_handler_knights)
diff --git a/sys/arch/amd64/amd64/spl.S b/sys/arch/amd64/amd64/spl.S
index c4782552d5c..84c6c6f5450 100644
--- a/sys/arch/amd64/amd64/spl.S
+++ b/sys/arch/amd64/amd64/spl.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: spl.S,v 1.17 2018/07/10 16:01:26 deraadt Exp $	*/
+/*	$OpenBSD: spl.S,v 1.18 2021/09/04 22:15:33 bluhm Exp $	*/
 /*	$NetBSD: spl.S,v 1.3 2004/06/28 09:13:11 fvdl Exp $	*/
 
 /*
@@ -89,6 +89,7 @@ _C_LABEL(splhigh):
 	xchgl	%eax,CPUVAR(ILEVEL)
 	RETGUARD_CHECK(splhigh, r11)
 	ret
+	lfence
 
 	.align 16, 0xcc
 _C_LABEL(splx):
@@ -99,6 +100,7 @@ _C_LABEL(splx):
 	jnz	_C_LABEL(Xspllower)
 	RETGUARD_CHECK(splx, r11)
 	ret
+	lfence
 #endif /* PROF || GPROF */
 #endif
 
@@ -136,6 +138,7 @@ KIDTVEC(spllower)
 	popq	%rbx
 	RETGUARD_CHECK(Xspllower, r11)
 	ret
+	lfence
 2:	bsrq	%rax,%rax
 	btrq	%rax,CPUVAR(IPENDING)
 	movq	CPUVAR(ISOURCES)(,%rax,8),%rax
diff --git a/sys/arch/amd64/amd64/vector.S b/sys/arch/amd64/amd64/vector.S
index cc8625bfe33..825a661477c 100644
--- a/sys/arch/amd64/amd64/vector.S
+++ b/sys/arch/amd64/amd64/vector.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vector.S,v 1.85 2021/09/03 16:45:44 jasper Exp $	*/
+/*	$OpenBSD: vector.S,v 1.86 2021/09/04 22:15:33 bluhm Exp $	*/
 /*	$NetBSD: vector.S,v 1.5 2004/06/28 09:13:11 fvdl Exp $	*/
 
 /*
@@ -537,6 +537,7 @@ KUENTRY(x2apic_eoi)
 	popq    %rcx
 	popq    %rax
 	ret
+	lfence
 END(x2apic_eoi)
 
 #if NLAPIC > 0
diff --git a/sys/arch/amd64/amd64/vmm_support.S b/sys/arch/amd64/amd64/vmm_support.S
index c328fb138ed..3be2f292a8a 100644
--- a/sys/arch/amd64/amd64/vmm_support.S
+++ b/sys/arch/amd64/amd64/vmm_support.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmm_support.S,v 1.17 2021/02/13 07:47:37 mlarkin Exp $	*/
+/*	$OpenBSD: vmm_support.S,v 1.18 2021/09/04 22:15:33 bluhm Exp $	*/
 /*
  * Copyright (c) 2014 Mike Larkin <mlarkin@openbsd.org>
  *
@@ -58,6 +58,7 @@ _C_LABEL(vmm_dispatch_intr):
 	cli
 	callq	*%rdi
 	ret
+	lfence
 
 _C_LABEL(vmxon):
 	RETGUARD_SETUP(vmxon, r11)
@@ -67,10 +68,12 @@ _C_LABEL(vmxon):
 	xorq	%rax, %rax
 	RETGUARD_CHECK(vmxon, r11)
 	ret
+	lfence
 failed_on:
 	movq	$0x01, %rax
 	RETGUARD_CHECK(vmxon, r11)
 	ret
+	lfence
 
 _C_LABEL(vmxoff):
 	RETGUARD_SETUP(vmxoff, r11)
@@ -80,10 +83,12 @@ _C_LABEL(vmxoff):
 	xorq	%rax, %rax
 	RETGUARD_CHECK(vmxoff, r11)
 	ret
+	lfence
 failed_off:
 	movq	$0x01, %rax
 	RETGUARD_CHECK(vmxoff, r11)
 	ret
+	lfence
 
 _C_LABEL(vmclear):
 	RETGUARD_SETUP(vmclear, r11)
@@ -93,10 +98,12 @@ _C_LABEL(vmclear):
 	xorq	%rax, %rax
 	RETGUARD_CHECK(vmclear, r11)
 	ret
+	lfence
 failed_clear:
 	movq	$0x01, %rax
 	RETGUARD_CHECK(vmclear, r11)
 	ret
+	lfence
 
 _C_LABEL(vmptrld):
 	RETGUARD_SETUP(vmptrld, r11)
@@ -106,10 +113,12 @@ _C_LABEL(vmptrld):
 	xorq	%rax, %rax
 	RETGUARD_CHECK(vmptrld, r11)
 	ret
+	lfence
 failed_ptrld:
 	movq	$0x01, %rax
 	RETGUARD_CHECK(vmptrld, r11)
 	ret
+	lfence
 
 _C_LABEL(vmptrst):
 	RETGUARD_SETUP(vmptrst, r11)
@@ -119,10 +128,12 @@ _C_LABEL(vmptrst):
 	xorq	%rax, %rax
 	RETGUARD_CHECK(vmptrst, r11)
 	ret
+	lfence
 failed_ptrst:
 	movq	$0x01, %rax
 	RETGUARD_CHECK(vmptrst, r11)
 	ret
+	lfence
 
 _C_LABEL(vmwrite):
 	RETGUARD_SETUP(vmwrite, r11)
@@ -132,10 +143,12 @@ _C_LABEL(vmwrite):
 	xorq	%rax, %rax
 	RETGUARD_CHECK(vmwrite, r11)
 	ret
+	lfence
 failed_write:
 	movq	$0x01, %rax
 	RETGUARD_CHECK(vmwrite, r11)
 	ret
+	lfence
 
 _C_LABEL(vmread):
 	RETGUARD_SETUP(vmread, r11)
@@ -145,22 +158,26 @@ _C_LABEL(vmread):
 	xorq	%rax, %rax
 	RETGUARD_CHECK(vmread, r11)
 	ret
+	lfence
 failed_read:
 	movq	$0x01, %rax
 	RETGUARD_CHECK(vmread, r11)
 	ret
+	lfence
 
 _C_LABEL(invvpid):
 	RETGUARD_SETUP(invvpid, r11)
 	invvpid (%rsi), %rdi
 	RETGUARD_CHECK(invvpid, r11)
 	ret
+	lfence
 
 _C_LABEL(invept):
 	RETGUARD_SETUP(invept, r11)
 	invept (%rsi), %rdi
 	RETGUARD_CHECK(invept, r11)
 	ret
+	lfence
 
 _C_LABEL(vmx_enter_guest):
 	RETGUARD_SETUP(vmx_enter_guest, r11)
@@ -534,6 +551,7 @@ restore_host:
 	movq	%rdi, %rax
 	RETGUARD_CHECK(vmx_enter_guest, r11)
 	ret
+	lfence
 
 _C_LABEL(svm_enter_guest):
 	RETGUARD_SETUP(svm_enter_guest, r11)
@@ -761,3 +779,4 @@ restore_host_svm:
 
 	RETGUARD_CHECK(svm_enter_guest, r11)
 	ret
+	lfence
diff --git a/sys/arch/amd64/include/asm.h b/sys/arch/amd64/include/asm.h
index fc575541d82..e642072c61a 100644
--- a/sys/arch/amd64/include/asm.h
+++ b/sys/arch/amd64/include/asm.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: asm.h,v 1.19 2021/09/01 09:50:21 bluhm Exp $	*/
+/*	$OpenBSD: asm.h,v 1.20 2021/09/04 22:15:33 bluhm Exp $	*/
 /*	$NetBSD: asm.h,v 1.2 2003/05/02 18:05:47 yamt Exp $	*/
 
 /*-
@@ -135,7 +135,7 @@
 	cmpq (__retguard_ ## x)(%rip), %reg; \
 	je 66f; \
 	int3; int3; \
-	.zero (0xf - ((. - x) & 0xf)), 0xcc; \
+	.zero (0xf - ((. + 3 - x) & 0xf)), 0xcc; \
 66:
 # define RETGUARD_PUSH(reg) \
 	pushq %reg
diff --git a/sys/arch/amd64/include/profile.h b/sys/arch/amd64/include/profile.h
index 9cb638f6ad2..7f63b92c8f0 100644
--- a/sys/arch/amd64/include/profile.h
+++ b/sys/arch/amd64/include/profile.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: profile.h,v 1.4 2012/08/22 17:19:35 pascal Exp $	*/
+/*	$OpenBSD: profile.h,v 1.5 2021/09/04 22:15:33 bluhm Exp $	*/
 /*	$NetBSD: profile.h,v 1.3 2003/11/28 23:22:45 fvdl Exp $	*/
 
 /*
@@ -67,6 +67,7 @@ __asm(" .globl __mcount		\n"			\
 "	movq	48(%rsp),%rax	\n"			\
 "	leave			\n"			\
 "	ret			\n"			\
+"	lfence			\n"			\
 "	.size __mcount,.-__mcount");