Implement support for AVX-512. This required some fixes to the so-far

unused Skylake AVX-512 MDS handler and increases the ci_mds_tmp array to
64 bytes.  With help from guenther@

ok deraadt@, guenther@

diff --git a/sys/arch/amd64/amd64/cpu.c b/sys/arch/amd64/amd64/cpu.c
index 78bd512..e4f355e 100644 (file)
--- a/sys/arch/amd64/amd64/cpu.c
+++ b/sys/arch/amd64/amd64/cpu.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: cpu.c,v 1.185 2024/04/03 02:01:21 guenther Exp $      */
+/*     $OpenBSD: cpu.c,v 1.186 2024/04/14 09:59:04 kettenis Exp $      */
 /* $NetBSD: cpu.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $ */
 
 /*-
@@ -294,6 +294,7 @@ replacemds(void)
        static int replacedone = 0;
        extern long mds_handler_bdw, mds_handler_ivb, mds_handler_skl;
        extern long mds_handler_skl_sse, mds_handler_skl_avx;
+       extern long mds_handler_skl_avx512;
        extern long mds_handler_silvermont, mds_handler_knights;
        struct cpu_info *ci = &cpu_info_primary;
        CPU_INFO_ITERATOR cii;
@@ -302,9 +303,9 @@ replacemds(void)
        int use_verw = 0, s;
        uint32_t cap = 0;
 
-       /* ci_mds_tmp must be 32byte aligned for AVX instructions */
+       /* ci_mds_tmp must be 64-byte aligned for AVX-512 instructions */
        CTASSERT((offsetof(struct cpu_info, ci_mds_tmp) -
-                 offsetof(struct cpu_info, ci_PAGEALIGN)) % 32 == 0);
+                 offsetof(struct cpu_info, ci_PAGEALIGN)) % 64 == 0);
 
        if (replacedone)
                return;
@@ -358,8 +359,10 @@ replacemds(void)
                         * Skylake, KabyLake, CoffeeLake, WhiskeyLake,
                         * CascadeLake
                         */
-                       /* XXX mds_handler_skl_avx512 */
-                       if (xgetbv(0) & XFEATURE_AVX) {
+                       if (xgetbv(0) & XFEATURE_AVX512) {
+                               handler = &mds_handler_skl_avx512;
+                               type = "Skylake AVX-512";
+                       } else if (xgetbv(0) & XFEATURE_AVX) {
                                handler = &mds_handler_skl_avx;
                                type = "Skylake AVX";
                        } else {
@@ -803,6 +806,7 @@ cpu_init(struct cpu_info *ci)
                xsave_mask = XFEATURE_X87 | XFEATURE_SSE;
                CPUID_LEAF(0xd, 0, eax, ebx, ecx, edx);
                xsave_mask |= eax & XFEATURE_AVX;
+               xsave_mask |= eax & XFEATURE_AVX512;
                xsetbv(0, xsave_mask);
                CPUID_LEAF(0xd, 0, eax, ebx, ecx, edx);
                if (CPU_IS_PRIMARY(ci)) {

diff --git a/sys/arch/amd64/amd64/mds.S b/sys/arch/amd64/amd64/mds.S
index e4967fb..630b7c1 100644 (file)
--- a/sys/arch/amd64/amd64/mds.S
+++ b/sys/arch/amd64/amd64/mds.S
@@ -1,4 +1,4 @@
-/*     $OpenBSD: mds.S,v 1.4 2023/04/22 18:27:28 guenther Exp $        */
+/*     $OpenBSD: mds.S,v 1.5 2024/04/14 09:59:04 kettenis Exp $        */
 /*
  * Copyright (c) 2019 Philip Guenther <guenther@openbsd.org>
  *
@@ -77,6 +77,8 @@ END(mds_handler_bdw)
 ENTRY(mds_handler_skl)
        xorl    %ecx,%ecx
        xgetbv
+       testb   $XFEATURE_AVX512,%al
+       jne     mds_handler_skl_avx512
        testb   $XFEATURE_AVX,%al
        jne     mds_handler_skl_avx
        jmp     mds_handler_skl_sse
@@ -136,14 +138,12 @@ ENTRY(mds_handler_skl_avx)
        lfence
 END(mds_handler_skl_avx)
 
-/* we don't support AVX512 yet */
-#if 0
 ENTRY(mds_handler_skl_avx512)
        RETGUARD_SETUP(mds_handler_skl_avx512, r11)
        movq    CPUVAR(MDS_BUF),%rax
        leaq    64(%rax),%rdi
        vmovdqa64       %zmm0,CPUVAR(MDS_TMP)
-       vpxor   %zmm0,%zmm0,%zmm0
+       vpxorq  %zmm0,%zmm0,%zmm0
 
        lfence
        vorpd   (%rax),%zmm0,%zmm0
@@ -164,7 +164,6 @@ ENTRY(mds_handler_skl_avx512)
        retq
        lfence
 END(mds_handler_skl_avx512)
-#endif
 
 ENTRY(mds_handler_silvermont)
        RETGUARD_SETUP(mds_handler_silvermont, r11)

diff --git a/sys/arch/amd64/include/cpu.h b/sys/arch/amd64/include/cpu.h
index 84a86b2..7bd9a54 100644 (file)
--- a/sys/arch/amd64/include/cpu.h
+++ b/sys/arch/amd64/include/cpu.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: cpu.h,v 1.164 2024/04/03 02:01:21 guenther Exp $      */
+/*     $OpenBSD: cpu.h,v 1.165 2024/04/14 09:59:04 kettenis Exp $      */
 /*     $NetBSD: cpu.h,v 1.1 2003/04/26 18:39:39 fvdl Exp $     */
 
 /*-
@@ -132,7 +132,7 @@ struct cpu_info {
        u_int64_t ci_user_cr3;          /* [o] U-K page table */
 
        /* bits for mitigating Micro-architectural Data Sampling */
-       char            ci_mds_tmp[32]; /* [o] 32byte aligned */
+       char            ci_mds_tmp[64]; /* [o] 64-byte aligned */
        void            *ci_mds_buf;    /* [I] */
 
        struct proc *ci_curproc;        /* [o] */

diff --git a/sys/arch/amd64/include/fpu.h b/sys/arch/amd64/include/fpu.h
index 2a024f2..f468815 100644 (file)
--- a/sys/arch/amd64/include/fpu.h
+++ b/sys/arch/amd64/include/fpu.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: fpu.h,v 1.19 2023/07/10 03:32:10 guenther Exp $       */
+/*     $OpenBSD: fpu.h,v 1.20 2024/04/14 09:59:04 kettenis Exp $       */
 /*     $NetBSD: fpu.h,v 1.1 2003/04/26 18:39:40 fvdl Exp $     */
 
 #ifndef        _MACHINE_FPU_H_
@@ -40,7 +40,7 @@ struct savefpu {
        struct fxsave64 fp_fxsave;      /* see above */
        struct xstate_hdr fp_xstate;
        u_int64_t fp_ymm[16][2];
-       u_int64_t fp_cet_u[2];
+       u_int8_t fp_components[1856];   /* enough for AVX-512 */
 };
 
 /*