-/* $OpenBSD: cpu.c,v 1.169 2023/06/15 22:18:06 cheloha Exp $ */
+/* $OpenBSD: cpu.c,v 1.170 2023/07/10 03:32:10 guenther Exp $ */
/* $NetBSD: cpu.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $ */
/*-
int cpu_apmi_edx = 0; /* cpuid(0x80000007).edx */
int ecpu_ecxfeature = 0; /* cpuid(0x80000001).ecx */
int cpu_meltdown = 0;
+int cpu_use_xsaves = 0;
void
replacesmap(void)
}
static void
-replacexsave(void)
+replacexsave(int xsave_ext)
{
- extern long _xrstor, _xsave, _xsaveopt;
- u_int32_t eax, ebx, ecx, edx;
+ extern long _xrstor, _xrstors, _xsave, _xsaves, _xsaveopt;
static int replacedone = 0;
int s;
return;
replacedone = 1;
- /* find out whether xsaveopt is supported */
- CPUID_LEAF(0xd, 1, eax, ebx, ecx, edx);
s = splhigh();
+ codepatch_replace(CPTAG_XRSTORS,
+ (xsave_ext & XSAVE_XSAVES) ? &_xrstors : &_xrstor, 4);
codepatch_replace(CPTAG_XRSTOR, &_xrstor, 4);
codepatch_replace(CPTAG_XSAVE,
- (eax & XSAVE_XSAVEOPT) ? &_xsaveopt : &_xsave, 4);
+ (xsave_ext & XSAVE_XSAVES) ? &_xsaves :
+ (xsave_ext & XSAVE_XSAVEOPT) ? &_xsaveopt : &_xsave, 4);
splx(s);
}
KASSERT(ebx == fpu_save_len);
}
- replacexsave();
+ /* check for xsaves, xsaveopt, and supervisor features */
+ CPUID_LEAF(0xd, 1, eax, ebx, ecx, edx);
+ /* Disable XSAVES on AMD family 17h due to Erratum 1386 */
+ if (!strcmp(cpu_vendor, "AuthenticAMD") &&
+ ci->ci_family == 0x17) {
+ eax &= ~XSAVE_XSAVES;
+ }
+ if (eax & XSAVE_XSAVES) {
+#ifndef SMALL_KERNEL
+ if (ci->ci_feature_sefflags_edx & SEFF0EDX_IBT)
+ xsave_mask |= ecx & XFEATURE_CET_U;
+#endif
+ if (xsave_mask & XFEATURE_XSS_MASK) {
+ wrmsr(MSR_XSS, xsave_mask & XFEATURE_XSS_MASK);
+ CPUID_LEAF(0xd, 1, eax, ebx, ecx, edx);
+ KASSERT(ebx <= sizeof(struct savefpu));
+ }
+ if (CPU_IS_PRIMARY(ci))
+ cpu_use_xsaves = 1;
+ }
+
+ replacexsave(eax);
}
- /* Give proc0 a clean FPU save area */
- sfp = &proc0.p_addr->u_pcb.pcb_savefpu;
- memset(sfp, 0, fpu_save_len);
- sfp->fp_fxsave.fx_fcw = __INITIAL_NPXCW__;
- sfp->fp_fxsave.fx_mxcsr = __INITIAL_MXCSR__;
- fpureset();
- if (xsave_mask) {
- /* must not use xsaveopt here */
- xsave(sfp, xsave_mask);
- } else
- fxsave(sfp);
+ if (CPU_IS_PRIMARY(ci)) {
+ /* Clean our FPU save area */
+ sfp = fpu_cleandata;
+ memset(sfp, 0, fpu_save_len);
+ sfp->fp_fxsave.fx_fcw = __INITIAL_NPXCW__;
+ sfp->fp_fxsave.fx_mxcsr = __INITIAL_MXCSR__;
+ xrstor_user(sfp, xsave_mask);
+ if (cpu_use_xsaves || !xsave_mask)
+ fpusave(sfp);
+ else {
+ /* must not use xsaveopt here */
+ xsave(sfp, xsave_mask);
+ }
+ } else {
+ fpureset();
+ }
#if NVMM > 0
/* Re-enable VMM if needed */
-/* $OpenBSD: locore.S,v 1.135 2023/07/05 18:23:10 anton Exp $ */
+/* $OpenBSD: locore.S,v 1.136 2023/07/10 03:32:10 guenther Exp $ */
/* $NetBSD: locore.S,v 1.13 2004/03/25 18:33:17 drochner Exp $ */
/*
#endif
CODEPATCH_START
fxrstor64 (%rdi)
- CODEPATCH_END(CPTAG_XRSTOR)
+ CODEPATCH_END(CPTAG_XRSTORS)
andl $~CPUPF_USERXSTATE,CPUVAR(PFLAGS)
.Lxstate_reset:
/* untouched state so can't fault */
CODEPATCH_START
fxrstor64 (%rdi)
- CODEPATCH_END(CPTAG_XRSTOR)
+ CODEPATCH_END(CPTAG_XRSTORS)
#if PCB_SAVEFPU != 0
subq $PCB_SAVEFPU,%rdi
#endif
#if PCB_SAVEFPU != 0
addq $PCB_SAVEFPU,%rdi
#endif
- movq xsave_mask(%rip),%rsi
- call xrstor_user
- testl %eax,%eax
- jnz .Lintr_xrstor_faulted
+ movq xsave_mask(%rip),%rdx
+ movl %edx,%eax
+ shrq $32, %rdx
+ CODEPATCH_START
+ fxrstor64 (%rdi)
+ CODEPATCH_END(CPTAG_XRSTORS)
+ //testl %eax,%eax
+ //jnz .Lintr_xrstor_faulted
.Lintr_restore_fsbase: /* CPU doesn't have curproc's FS.base */
orl $CPUPF_USERSEGS,CPUVAR(PFLAGS)
movq CPUVAR(CURPCB),%rdx
#endif
CODEPATCH_START
fxrstor64 (%rdi)
- CODEPATCH_END(CPTAG_XRSTOR)
+ CODEPATCH_END(CPTAG_XRSTORS)
movq $T_PROTFLT,TF_TRAPNO(%rsp)
jmp recall_trap
testq $PSL_I,%rdx
jnz .Lintr_exit_not_blocked
#endif /* DIAGNOSTIC */
- call pku_xonly /* XXX guenther disapproves, but foo3 locks */
movq TF_RDI(%rsp),%rdi
movq TF_RSI(%rsp),%rsi
movq TF_R8(%rsp),%r8
/*
* FPU/"extended CPU state" handling
+ * void xrstor_kern(sfp, mask)
+ * using first of xrstors/xrstor/fxrstor, load given state
+ * which is assumed to be trusted: i.e., unaltered from
+ * xsaves/xsaveopt/xsave/fxsave by kernel
* int xrstor_user(sfp, mask)
- * load given state, returns 0/1 if okay/it trapped
+ * using first of xrstor/fxrstor, load given state which might
+ * not be trustable: #GP faults will be caught; returns 0/1 if
+ * okay/it trapped.
* void fpusave(sfp)
* save current state, but retain it in the FPU
* void fpusavereset(sfp)
* load specified %xcr# register, returns 0/1 if okay/it trapped
*/
+ENTRY(xrstor_kern)
+ RETGUARD_SETUP(xrstor_kern, r11)
+ movq %rsi, %rdx
+ movl %esi, %eax
+ shrq $32, %rdx
+ CODEPATCH_START
+ fxrstor64 (%rdi)
+ CODEPATCH_END(CPTAG_XRSTORS)
+ RETGUARD_CHECK(xrstor_kern, r11)
+ ret
+ lfence
+END(xrstor_kern)
+
ENTRY(xrstor_user)
RETGUARD_SETUP(xrstor_user, r11)
movq %rsi, %rdx
#endif
CODEPATCH_START
fxrstor64 (%rdi)
- CODEPATCH_END(CPTAG_XRSTOR)
+ CODEPATCH_END(CPTAG_XRSTORS)
RETGUARD_CHECK(fpusavereset, r11)
ret
lfence
_xrstor:
xrstor64 (%rdi)
+ .globl _xrstors
+_xrstors:
+ xrstors64 (%rdi)
+
.globl _xsave
_xsave:
- xsave64 (%rdi)
+ xsave64 (%rdi)
+
+ .globl _xsaves
+_xsaves:
+ xsaves64 (%rdi)
.globl _xsaveopt
_xsaveopt:
-/* $OpenBSD: machdep.c,v 1.284 2022/11/29 21:41:39 guenther Exp $ */
+/* $OpenBSD: machdep.c,v 1.285 2023/07/10 03:32:10 guenther Exp $ */
/* $NetBSD: machdep.c,v 1.3 2003/05/07 22:58:18 fvdl Exp $ */
/*-
/* NOTREACHED */
}
+static inline void
+maybe_enable_user_cet(struct proc *p)
+{
+#ifndef SMALL_KERNEL
+ /* Enable indirect-branch tracking if present and not disabled */
+ if ((xsave_mask & XFEATURE_CET_U) &&
+ (p->p_p->ps_flags & PS_NOBTCFI) == 0) {
+ uint64_t msr = rdmsr(MSR_U_CET);
+ wrmsr(MSR_U_CET, msr | MSR_CET_ENDBR_EN | MSR_CET_NO_TRACK_EN);
+ }
+#endif
+}
+
+static inline void
+initialize_thread_xstate(struct proc *p)
+{
+ if (cpu_use_xsaves) {
+ xrstors(fpu_cleandata, xsave_mask);
+ maybe_enable_user_cet(p);
+ } else {
+ /* Reset FPU state in PCB */
+ memcpy(&p->p_addr->u_pcb.pcb_savefpu, fpu_cleandata,
+ fpu_save_len);
+
+ if (curcpu()->ci_pflags & CPUPF_USERXSTATE) {
+ /* state in CPU is obsolete; reset it */
+ fpureset();
+ }
+ }
+
+ /* The reset state _is_ the userspace state for this thread now */
+ curcpu()->ci_pflags |= CPUPF_USERXSTATE;
+}
+
+/*
+ * Copy out the FPU state, massaging it to be usable from userspace
+ * and acceptable to xrstor_user()
+ */
+static inline int
+copyoutfpu(struct savefpu *sfp, char *sp, size_t len)
+{
+ uint64_t bvs[2];
+
+ if (copyout(sfp, sp, len))
+ return 1;
+ if (len > offsetof(struct savefpu, fp_xstate.xstate_bv)) {
+ sp += offsetof(struct savefpu, fp_xstate.xstate_bv);
+ len -= offsetof(struct savefpu, fp_xstate.xstate_bv);
+ bvs[0] = sfp->fp_xstate.xstate_bv & XFEATURE_XCR0_MASK;
+ bvs[1] = sfp->fp_xstate.xstate_xcomp_bv &
+ (XFEATURE_XCR0_MASK | XFEATURE_COMPRESSED);
+ if (copyout(bvs, sp, min(len, sizeof bvs)))
+ return 1;
+ }
+ return 0;
+}
+
/*
* Send an interrupt to process.
*
else
sp = tf->tf_rsp - 128;
- sp &= ~15ULL; /* just in case */
- sss = (sizeof(ksc) + 15) & ~15;
+ sp -= fpu_save_len;
+ if (cpu_use_xsaves)
+ sp &= ~63ULL; /* just in case */
+ else
+ sp &= ~15ULL; /* just in case */
/* Save FPU state to PCB if necessary, then copy it out */
- if (curcpu()->ci_pflags & CPUPF_USERXSTATE) {
- curcpu()->ci_pflags &= ~CPUPF_USERXSTATE;
- fpusavereset(&p->p_addr->u_pcb.pcb_savefpu);
- }
- sp -= fpu_save_len;
- ksc.sc_fpstate = (struct fxsave64 *)sp;
- if (copyout(sfp, (void *)sp, fpu_save_len))
+ if (curcpu()->ci_pflags & CPUPF_USERXSTATE)
+ fpusave(&p->p_addr->u_pcb.pcb_savefpu);
+ if (copyoutfpu(sfp, (void *)sp, fpu_save_len))
return 1;
- /* Now reset the FPU state in PCB */
- memcpy(&p->p_addr->u_pcb.pcb_savefpu,
- &proc0.p_addr->u_pcb.pcb_savefpu, fpu_save_len);
+ initialize_thread_xstate(p);
+ ksc.sc_fpstate = (struct fxsave64 *)sp;
+ sss = (sizeof(ksc) + 15) & ~15;
sip = 0;
if (info) {
sip = sp - ((sizeof(*ksip) + 15) & ~15);
tf->tf_rsp = scp;
tf->tf_ss = GSEL(GUDATA_SEL, SEL_UPL);
- /* The reset state _is_ the userspace state for this thread now */
- curcpu()->ci_pflags |= CPUPF_USERXSTATE;
-
return 0;
}
} */ *uap = v;
struct sigcontext ksc, *scp = SCARG(uap, sigcntxp);
struct trapframe *tf = p->p_md.md_regs;
+ struct savefpu *sfp = &p->p_addr->u_pcb.pcb_savefpu;
int error;
if (PROC_PC(p) != p->p_p->ps_sigcoderet) {
!USERMODE(ksc.sc_cs, ksc.sc_eflags))
return (EINVAL);
- /* Current state is obsolete; toss it and force a reload */
+ /* Current FPU state is obsolete; toss it and force a reload */
if (curcpu()->ci_pflags & CPUPF_USERXSTATE) {
curcpu()->ci_pflags &= ~CPUPF_USERXSTATE;
fpureset();
/* Copy in the FPU state to restore */
if (__predict_true(ksc.sc_fpstate != NULL)) {
- struct fxsave64 *fx = &p->p_addr->u_pcb.pcb_savefpu.fp_fxsave;
-
- if ((error = copyin(ksc.sc_fpstate, fx, fpu_save_len)))
- return (error);
- fx->fx_mxcsr &= fpu_mxcsr_mask;
+ if ((error = copyin(ksc.sc_fpstate, sfp, fpu_save_len)))
+ return error;
+ if (xrstor_user(sfp, xsave_mask)) {
+ memcpy(sfp, fpu_cleandata, fpu_save_len);
+ return EINVAL;
+ }
+ maybe_enable_user_cet(p);
+ curcpu()->ci_pflags |= CPUPF_USERXSTATE;
} else {
/* shouldn't happen, but handle it */
- memcpy(&p->p_addr->u_pcb.pcb_savefpu,
- &proc0.p_addr->u_pcb.pcb_savefpu, fpu_save_len);
+ initialize_thread_xstate(p);
}
tf->tf_rdi = ksc.sc_rdi;
{
struct trapframe *tf;
- /* Reset FPU state in PCB */
- memcpy(&p->p_addr->u_pcb.pcb_savefpu,
- &proc0.p_addr->u_pcb.pcb_savefpu, fpu_save_len);
-
- if (curcpu()->ci_pflags & CPUPF_USERXSTATE) {
- /* state in CPU is obsolete; reset it */
- fpureset();
- } else {
- /* the reset state _is_ the userspace state now */
- curcpu()->ci_pflags |= CPUPF_USERXSTATE;
- }
+ initialize_thread_xstate(p);
/* To reset all registers we have to return via iretq */
p->p_md.md_flags |= MDP_IRET;
-/* $OpenBSD: vmm_machdep.c,v 1.3 2023/04/26 15:40:51 mlarkin Exp $ */
+/* $OpenBSD: vmm_machdep.c,v 1.4 2023/07/10 03:32:10 guenther Exp $ */
/*
* Copyright (c) 2014 Mike Larkin <mlarkin@openbsd.org>
*
fpusavereset(&curproc->p_addr->u_pcb.pcb_savefpu);
}
- if (vcpu->vc_fpuinited) {
- if (xrstor_user(&vcpu->vc_g_fpu, xsave_mask)) {
- DPRINTF("%s: guest attempted to set invalid %s\n",
- __func__, "xsave/xrstor state");
- return EINVAL;
- }
- }
+ if (vcpu->vc_fpuinited)
+ xrstor_kern(&vcpu->vc_g_fpu, xsave_mask);
if (xsave_mask) {
/* Restore guest %xcr0 */
vcpu->vc_gueststate.vg_xcr0 = xgetbv(0);
/* Restore host %xcr0 */
- xsetbv(0, xsave_mask);
+ xsetbv(0, xsave_mask & XFEATURE_XCR0_MASK);
}
/*
-/* $OpenBSD: codepatch.h,v 1.14 2020/03/11 07:27:08 guenther Exp $ */
+/* $OpenBSD: codepatch.h,v 1.15 2023/07/10 03:32:10 guenther Exp $ */
/*
* Copyright (c) 2014-2015 Stefan Fritsch <sf@sfritsch.de>
*
#define CPTAG_MDS_VMM 10
#define CPTAG_FENCE_SWAPGS_MIS_TAKEN 11
#define CPTAG_FENCE_NO_SAFE_SMAP 12
+#define CPTAG_XRSTORS 13
/*
* stac/clac SMAP instructions have lfence like semantics. Let's
-/* $OpenBSD: fpu.h,v 1.18 2023/05/22 00:39:57 guenther Exp $ */
+/* $OpenBSD: fpu.h,v 1.19 2023/07/10 03:32:10 guenther Exp $ */
/* $NetBSD: fpu.h,v 1.1 2003/04/26 18:39:40 fvdl Exp $ */
#ifndef _MACHINE_FPU_H_
struct fxsave64 fp_fxsave; /* see above */
struct xstate_hdr fp_xstate;
u_int64_t fp_ymm[16][2];
+ u_int64_t fp_cet_u[2];
};
/*
extern size_t fpu_save_len;
extern uint32_t fpu_mxcsr_mask;
extern uint64_t xsave_mask;
+extern int cpu_use_xsaves;
void fpuinit(struct cpu_info *);
int fputrap(int _type);
void fpu_kernel_enter(void);
void fpu_kernel_exit(void);
+/* pointer to fxsave/xsave/xsaves data with everything reset */
+#define fpu_cleandata (&proc0.p_addr->u_pcb.pcb_savefpu)
+
int xrstor_user(struct savefpu *_addr, uint64_t _mask);
+void xrstor_kern(struct savefpu *_addr, uint64_t _mask);
#define fpureset() \
- xrstor_user(&proc0.p_addr->u_pcb.pcb_savefpu, xsave_mask)
+ xrstor_kern(fpu_cleandata, xsave_mask)
int xsetbv_user(uint32_t _reg, uint64_t _mask);
#define fninit() __asm("fninit")
lo = mask;
hi = mask >> 32;
- /* should be xsave64, but where we use this it doesn't matter */
- __asm volatile("xsave %0" : "=m" (*addr) : "a" (lo), "d" (hi) :
- "memory");
+ __asm volatile("xsave64 %0" : "+m" (*addr) : "a" (lo), "d" (hi));
+}
+
+static inline void
+xrstors(const struct savefpu *addr, uint64_t mask)
+{
+ uint32_t lo, hi;
+
+ lo = mask;
+ hi = mask >> 32;
+ __asm volatile("xrstors64 %0" : : "m" (*addr), "a" (lo), "d" (hi));
}
#endif
-/* $OpenBSD: specialreg.h,v 1.102 2023/04/22 18:27:28 guenther Exp $ */
+/* $OpenBSD: specialreg.h,v 1.103 2023/07/10 03:32:10 guenther Exp $ */
/* $NetBSD: specialreg.h,v 1.1 2003/04/26 18:39:48 fvdl Exp $ */
/* $NetBSD: x86/specialreg.h,v 1.2 2003/04/25 21:54:30 fvdl Exp $ */
#define XFEATURE_TILEDATA 0x00040000 /* AMX state */
#define XFEATURE_AMX (XFEATURE_TILEDATA | XFEATURE_TILEDATA)
+/* valid only in xcomp_bv field: */
+#define XFEATURE_COMPRESSED (1ULL << 63) /* compressed format */
+
/* which bits are for XCR0 and which for the XSS MSR? */
#define XFEATURE_XCR0_MASK \
(XFEATURE_X87 | XFEATURE_SSE | XFEATURE_AVX | XFEATURE_MPX | \
#define MSR_MC3_MISC 0x413
#define MSR_U_CET 0x6a0
#define MSR_CET_ENDBR_EN (1 << 2)
+#define MSR_CET_NO_TRACK_EN (1 << 4)
#define MSR_S_CET 0x6a2
#define MSR_PKRS 0x6e1
#define MSR_XSS 0xda0
projects / openbsd / commitdiff