From c0f33c9875c4ab47e986b698610630b6cbf21c6c Mon Sep 17 00:00:00 2001 From: kettenis Date: Sun, 14 Apr 2024 09:59:04 +0000 Subject: [PATCH] Implement support for AVX-512. This required some fixes to the so-far unused Skylake AVX-512 MDS handler and increases the ci_mds_tmp array to 64 bytes. With help from guenther@ ok deraadt@, guenther@ --- sys/arch/amd64/amd64/cpu.c | 14 +++++++++----- sys/arch/amd64/amd64/mds.S | 9 ++++----- sys/arch/amd64/include/cpu.h | 4 ++-- sys/arch/amd64/include/fpu.h | 4 ++-- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/sys/arch/amd64/amd64/cpu.c b/sys/arch/amd64/amd64/cpu.c index 78bd512da93..e4f355e12b5 100644 --- a/sys/arch/amd64/amd64/cpu.c +++ b/sys/arch/amd64/amd64/cpu.c @@ -1,4 +1,4 @@ -/* $OpenBSD: cpu.c,v 1.185 2024/04/03 02:01:21 guenther Exp $ */ +/* $OpenBSD: cpu.c,v 1.186 2024/04/14 09:59:04 kettenis Exp $ */ /* $NetBSD: cpu.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $ */ /*- @@ -294,6 +294,7 @@ replacemds(void) static int replacedone = 0; extern long mds_handler_bdw, mds_handler_ivb, mds_handler_skl; extern long mds_handler_skl_sse, mds_handler_skl_avx; + extern long mds_handler_skl_avx512; extern long mds_handler_silvermont, mds_handler_knights; struct cpu_info *ci = &cpu_info_primary; CPU_INFO_ITERATOR cii; @@ -302,9 +303,9 @@ replacemds(void) int use_verw = 0, s; uint32_t cap = 0; - /* ci_mds_tmp must be 32byte aligned for AVX instructions */ + /* ci_mds_tmp must be 64-byte aligned for AVX-512 instructions */ CTASSERT((offsetof(struct cpu_info, ci_mds_tmp) - - offsetof(struct cpu_info, ci_PAGEALIGN)) % 32 == 0); + offsetof(struct cpu_info, ci_PAGEALIGN)) % 64 == 0); if (replacedone) return; @@ -358,8 +359,10 @@ replacemds(void) * Skylake, KabyLake, CoffeeLake, WhiskeyLake, * CascadeLake */ - /* XXX mds_handler_skl_avx512 */ - if (xgetbv(0) & XFEATURE_AVX) { + if (xgetbv(0) & XFEATURE_AVX512) { + handler = &mds_handler_skl_avx512; + type = "Skylake AVX-512"; + } else if (xgetbv(0) & XFEATURE_AVX) { handler = &mds_handler_skl_avx; type = "Skylake AVX"; } else { @@ -803,6 +806,7 @@ cpu_init(struct cpu_info *ci) xsave_mask = XFEATURE_X87 | XFEATURE_SSE; CPUID_LEAF(0xd, 0, eax, ebx, ecx, edx); xsave_mask |= eax & XFEATURE_AVX; + xsave_mask |= eax & XFEATURE_AVX512; xsetbv(0, xsave_mask); CPUID_LEAF(0xd, 0, eax, ebx, ecx, edx); if (CPU_IS_PRIMARY(ci)) { diff --git a/sys/arch/amd64/amd64/mds.S b/sys/arch/amd64/amd64/mds.S index e4967fbffe9..630b7c17074 100644 --- a/sys/arch/amd64/amd64/mds.S +++ b/sys/arch/amd64/amd64/mds.S @@ -1,4 +1,4 @@ -/* $OpenBSD: mds.S,v 1.4 2023/04/22 18:27:28 guenther Exp $ */ +/* $OpenBSD: mds.S,v 1.5 2024/04/14 09:59:04 kettenis Exp $ */ /* * Copyright (c) 2019 Philip Guenther * @@ -77,6 +77,8 @@ END(mds_handler_bdw) ENTRY(mds_handler_skl) xorl %ecx,%ecx xgetbv + testb $XFEATURE_AVX512,%al + jne mds_handler_skl_avx512 testb $XFEATURE_AVX,%al jne mds_handler_skl_avx jmp mds_handler_skl_sse @@ -136,14 +138,12 @@ ENTRY(mds_handler_skl_avx) lfence END(mds_handler_skl_avx) -/* we don't support AVX512 yet */ -#if 0 ENTRY(mds_handler_skl_avx512) RETGUARD_SETUP(mds_handler_skl_avx512, r11) movq CPUVAR(MDS_BUF),%rax leaq 64(%rax),%rdi vmovdqa64 %zmm0,CPUVAR(MDS_TMP) - vpxor %zmm0,%zmm0,%zmm0 + vpxorq %zmm0,%zmm0,%zmm0 lfence vorpd (%rax),%zmm0,%zmm0 @@ -164,7 +164,6 @@ ENTRY(mds_handler_skl_avx512) retq lfence END(mds_handler_skl_avx512) -#endif ENTRY(mds_handler_silvermont) RETGUARD_SETUP(mds_handler_silvermont, r11) diff --git a/sys/arch/amd64/include/cpu.h b/sys/arch/amd64/include/cpu.h index 84a86b229da..7bd9a54fa0f 100644 --- a/sys/arch/amd64/include/cpu.h +++ b/sys/arch/amd64/include/cpu.h @@ -1,4 +1,4 @@ -/* $OpenBSD: cpu.h,v 1.164 2024/04/03 02:01:21 guenther Exp $ */ +/* $OpenBSD: cpu.h,v 1.165 2024/04/14 09:59:04 kettenis Exp $ */ /* $NetBSD: cpu.h,v 1.1 2003/04/26 18:39:39 fvdl Exp $ */ /*- @@ -132,7 +132,7 @@ struct cpu_info { u_int64_t ci_user_cr3; /* [o] U-K page table */ /* bits for mitigating Micro-architectural Data Sampling */ - char ci_mds_tmp[32]; /* [o] 32byte aligned */ + char ci_mds_tmp[64]; /* [o] 64-byte aligned */ void *ci_mds_buf; /* [I] */ struct proc *ci_curproc; /* [o] */ diff --git a/sys/arch/amd64/include/fpu.h b/sys/arch/amd64/include/fpu.h index 2a024f2dce4..f468815ff76 100644 --- a/sys/arch/amd64/include/fpu.h +++ b/sys/arch/amd64/include/fpu.h @@ -1,4 +1,4 @@ -/* $OpenBSD: fpu.h,v 1.19 2023/07/10 03:32:10 guenther Exp $ */ +/* $OpenBSD: fpu.h,v 1.20 2024/04/14 09:59:04 kettenis Exp $ */ /* $NetBSD: fpu.h,v 1.1 2003/04/26 18:39:40 fvdl Exp $ */ #ifndef _MACHINE_FPU_H_ @@ -40,7 +40,7 @@ struct savefpu { struct fxsave64 fp_fxsave; /* see above */ struct xstate_hdr fp_xstate; u_int64_t fp_ymm[16][2]; - u_int64_t fp_cet_u[2]; + u_int8_t fp_components[1856]; /* enough for AVX-512 */ }; /* -- 2.20.1