Do "Return stack refilling", based on the "Return stack underflow" discussion

and its associated appendix at https://support.google.com/faqs/answer/7625886
This should address at least some cases of "SpectreRSB" and earlier
Spectre variants; more commits to follow.

The refilling is done in the enter-kernel-from-userspace and
return-to-userspace-from-kernel paths, making sure to do it before
unblocking interrupts so that a successive interrupt can't get the
CPU to C code without doing this refill.  Per the link above, it
also does it immediately after mwait, apparently in case the low-power
CPU states of idle-via-mwait flush the RSB.

ok mlarkin@ deraadt@

diff --git a/sys/arch/amd64/amd64/locore.S b/sys/arch/amd64/amd64/locore.S
index e3cfbdc..1318b32 100644 (file)
--- a/sys/arch/amd64/amd64/locore.S
+++ b/sys/arch/amd64/amd64/locore.S
@@ -1,4 +1,4 @@
-/*     $OpenBSD: locore.S,v 1.105 2018/07/12 14:11:11 guenther Exp $   */
+/*     $OpenBSD: locore.S,v 1.106 2018/07/23 17:54:04 guenther Exp $   */
 /*     $NetBSD: locore.S,v 1.13 2004/03/25 18:33:17 drochner Exp $     */
 
 /*
@@ -585,6 +585,9 @@ IDTVEC_NOALIGN(syscall)
        movq    %rax,CPUVAR(SCRATCH)
        movq    CPUVAR(KERN_RSP),%rax
        xchgq   %rax,%rsp
+       movq    %rcx,TF_RCX(%rsp)
+       movq    %rcx,TF_RIP(%rsp)
+       RET_STACK_REFILL_WITH_RCX
        sti
 
        /*
@@ -598,10 +601,8 @@ IDTVEC_NOALIGN(syscall)
        movq    %rax,TF_RSP(%rsp)
        movq    CPUVAR(SCRATCH),%rax
        INTR_SAVE_MOST_GPRS_NO_ADJ
-       movq    %rcx,TF_RCX(%rsp)
        movq    %r11, TF_RFLAGS(%rsp)   /* old rflags from syscall insn */
        movq    $(GSEL(GUCODE_SEL, SEL_UPL)), TF_CS(%rsp)
-       movq    %rcx,TF_RIP(%rsp)
        movq    %rax,TF_ERR(%rsp)       /* stash syscall # for SPL check */
        INTR_CLEAR_GPRS
 
@@ -641,6 +642,8 @@ IDTVEC_NOALIGN(syscall)
        jz      .Lsyscall_restore_fsbase
 
 .Lsyscall_restore_registers:
+       RET_STACK_REFILL_WITH_RCX
+
        movq    TF_RDI(%rsp),%rdi
        movq    TF_RSI(%rsp),%rsi
        movq    TF_R8(%rsp),%r8
@@ -804,6 +807,8 @@ intr_user_exit_post_ast:
        jz      .Lintr_restore_fsbase
 
 .Lintr_restore_registers:
+       RET_STACK_REFILL_WITH_RCX
+
        movq    TF_RDI(%rsp),%rdi
        movq    TF_RSI(%rsp),%rsi
        movq    TF_R8(%rsp),%r8

diff --git a/sys/arch/amd64/include/asm.h b/sys/arch/amd64/include/asm.h
index 5c29eae..583e888 100644 (file)
--- a/sys/arch/amd64/include/asm.h
+++ b/sys/arch/amd64/include/asm.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: asm.h,v 1.15 2018/07/12 14:11:11 guenther Exp $       */
+/*     $OpenBSD: asm.h,v 1.16 2018/07/23 17:54:04 guenther Exp $       */
 /*     $NetBSD: asm.h,v 1.2 2003/05/02 18:05:47 yamt Exp $     */
 
 /*-
@@ -90,6 +90,21 @@
 #define KUENTRY(x) \
        KUTEXT; _ALIGN_TRAPS; _GENTRY(x)
 
+/* Return stack refill, to prevent speculation attacks on natural returns */
+#define        RET_STACK_REFILL_WITH_RCX       \
+               mov     $8,%rcx         ; \
+               _ALIGN_TEXT             ; \
+       3:      call    5f              ; \
+       4:      pause                   ; \
+               call    4b              ; \
+               _ALIGN_TRAPS            ; \
+       5:      call    7f              ; \
+       6:      pause                   ; \
+               call    6b              ; \
+               _ALIGN_TRAPS            ; \
+       7:      loop    3b              ; \
+               add     $(16*8),%rsp
+
 #endif /* _KERNEL */
 
 #ifdef __STDC__

diff --git a/sys/arch/amd64/include/cpufunc.h b/sys/arch/amd64/include/cpufunc.h
index 60c9de7..7c7e391 100644 (file)
--- a/sys/arch/amd64/include/cpufunc.h
+++ b/sys/arch/amd64/include/cpufunc.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: cpufunc.h,v 1.26 2018/06/30 10:16:35 kettenis Exp $   */
+/*     $OpenBSD: cpufunc.h,v 1.27 2018/07/23 17:54:04 guenther Exp $   */
 /*     $NetBSD: cpufunc.h,v 1.3 2003/05/08 10:27:43 fvdl Exp $ */
 
 /*-
@@ -289,7 +289,21 @@ static __inline void
 mwait(u_long extensions, u_int hints)
 {
 
-       __asm volatile("mwait" : : "a" (hints), "c" (extensions));
+       __asm volatile(
+               "       mwait                   ;"
+               "       mov     $8,%%rcx        ;"
+               "       .align  16,0x90         ;"
+               "3:     call    5f              ;"
+               "4:     pause                   ;"
+               "       call    4b              ;"
+               "       .align  16,0xcc         ;"
+               "5:     call    7f              ;"
+               "6:     pause                   ;"
+               "       call    6b              ;"
+               "       .align  16,0xcc         ;"
+               "7:     loop    3b              ;"
+               "       add     $(16*8),%%rsp"
+           : "=c" (extensions) : "a" (hints));
 }
 
 static __inline void

diff --git a/sys/arch/amd64/include/frameasm.h b/sys/arch/amd64/include/frameasm.h
index 33ab8b4..3d4eade 100644 (file)
--- a/sys/arch/amd64/include/frameasm.h
+++ b/sys/arch/amd64/include/frameasm.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: frameasm.h,v 1.19 2018/07/21 02:19:54 guenther Exp $  */
+/*     $OpenBSD: frameasm.h,v 1.20 2018/07/23 17:54:04 guenther Exp $  */
 /*     $NetBSD: frameasm.h,v 1.1 2003/04/26 18:39:40 fvdl Exp $        */
 
 #ifndef _AMD64_MACHINE_FRAMEASM_H
@@ -93,6 +93,7 @@ INTRENTRY_LABEL(label):       /* from kernel */ \
        movq    CPUVAR(KERN_RSP),%rax   ; \
        xchgq   %rax,%rsp               ; \
        movq    %rcx,TF_RCX(%rsp)       ; \
+       RET_STACK_REFILL_WITH_RCX       ; \
        /* copy trapno+err to the trap frame */ \
        movq    0(%rax),%rcx            ; \
        movq    %rcx,TF_TRAPNO(%rsp)    ; \