From bb00e81153b2ff3fcf42c8955e50b70f573a8323 Mon Sep 17 00:00:00 2001
From: claudio <claudio@openbsd.org>
Date: Tue, 24 Oct 2023 13:20:09 +0000
Subject: [PATCH] Normally context switches happen in mi_switch() but there are
 3 cases where a switch happens outside. Cleanup these code paths and make the
 machine independent.

- when a process forks (fork, tfork, kthread), the new proc needs to
  somehow be scheduled for the first time. This is done by proc_trampoline.
  Since proc_trampoline is machine dependent assembler code change
  the MP specific proc_trampoline_mp() to proc_trampoline_mi() and make
  sure it is now always called.
- cpu_hatch: when booting APs the code needs to jump to the first proc
  running on that CPU. This should be the idle thread for that CPU.
- sched_exit: when a proc exits it needs to switch away from itself and
  then instruct the reaper to clean up the rest. This is done by switching
  to the idle loop.

Since the last two cases require a context switch to the idle proc factor
out the common code to sched_toidle() and use it in those places.

Tested by many on all archs.
OK miod@ mpi@ cheloha@
---
 sys/arch/alpha/alpha/cpu.c            |  7 +---
 sys/arch/alpha/alpha/locore.s         |  6 +--
 sys/arch/alpha/alpha/vm_machdep.c     | 10 +----
 sys/arch/amd64/amd64/cpu.c            |  7 +---
 sys/arch/amd64/amd64/locore.S         |  7 +---
 sys/arch/arm/arm/cpu.c                |  7 +---
 sys/arch/arm/arm/cpuswitch7.S         |  8 +---
 sys/arch/arm64/arm64/cpu.c            |  7 +---
 sys/arch/arm64/arm64/cpuswitch.S      | 10 +----
 sys/arch/hppa/dev/cpu.c               | 10 +----
 sys/arch/hppa/hppa/locore.S           |  8 +---
 sys/arch/i386/i386/cpu.c              |  6 +--
 sys/arch/i386/i386/locore.s           |  7 +---
 sys/arch/loongson/loongson/machdep.c  |  7 +---
 sys/arch/luna88k/luna88k/machdep.c    |  7 +---
 sys/arch/m88k/m88k/eh_common.S        |  9 +---
 sys/arch/macppc/macppc/cpu.c          | 11 ++---
 sys/arch/macppc/macppc/locore.S       |  8 +---
 sys/arch/macppc/macppc/machdep.c      |  9 +---
 sys/arch/mips64/mips64/clock.c        |  6 +--
 sys/arch/mips64/mips64/context.S      | 15 +------
 sys/arch/octeon/octeon/machdep.c      |  7 +---
 sys/arch/powerpc64/powerpc64/cpu.c    |  7 +---
 sys/arch/powerpc64/powerpc64/locore.S |  8 +---
 sys/arch/riscv64/riscv64/cpu.c        |  7 +---
 sys/arch/riscv64/riscv64/cpuswitch.S  | 10 +----
 sys/arch/sh/sh/locore_subr.S          |  7 +++-
 sys/arch/sh/sh/vm_machdep.c           |  5 ++-
 sys/arch/sparc64/sparc64/cpu.c        | 10 +----
 sys/arch/sparc64/sparc64/locore.s     |  9 ++--
 sys/kern/kern_fork.c                  | 34 +++++++++++----
 sys/kern/kern_sched.c                 | 59 +++++++++++++++------------
 sys/sys/sched.h                       |  3 +-
 33 files changed, 124 insertions(+), 214 deletions(-)

diff --git a/sys/arch/alpha/alpha/cpu.c b/sys/arch/alpha/alpha/cpu.c
index 03d035c3100..0b605f6af46 100644
--- a/sys/arch/alpha/alpha/cpu.c
+++ b/sys/arch/alpha/alpha/cpu.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: cpu.c,v 1.47 2023/06/15 22:18:06 cheloha Exp $ */
+/* $OpenBSD: cpu.c,v 1.48 2023/10/24 13:20:09 claudio Exp $ */
 /* $NetBSD: cpu.c,v 1.44 2000/05/23 05:12:53 thorpej Exp $ */
 
 /*-
@@ -569,7 +569,6 @@ cpu_hatch(struct cpu_info *ci)
 {
 	u_long cpu_id = cpu_number();
 	u_long cpumask = (1UL << cpu_id);
-	int s;
 
 	/* Mark the kernel pmap active on this processor. */
 	atomic_setbits_ulong(&pmap_kernel()->pm_cpus, cpumask);
@@ -600,7 +599,6 @@ cpu_hatch(struct cpu_info *ci)
 	clockqueue_init(&ci->ci_queue);
 	KERNEL_LOCK();
 	sched_init_cpu(ci);
-	nanouptime(&ci->ci_schedstate.spc_runtime);
 	ci->ci_curproc = ci->ci_fpcurproc = NULL;
 	ci->ci_randseed = (arc4random() & 0x7fffffff) + 1;
 	KERNEL_UNLOCK();
@@ -608,8 +606,7 @@ cpu_hatch(struct cpu_info *ci)
 	clockintr_cpu_init(NULL);
 
 	(void) alpha_pal_swpipl(ALPHA_PSL_IPL_0);
-	SCHED_LOCK(s);
-	cpu_switchto(NULL, sched_chooseproc());
+	sched_toidle();
 	/* NOTREACHED */
 }
 
diff --git a/sys/arch/alpha/alpha/locore.s b/sys/arch/alpha/alpha/locore.s
index 0c8f3cd1e64..959a6d0326c 100644
--- a/sys/arch/alpha/alpha/locore.s
+++ b/sys/arch/alpha/alpha/locore.s
@@ -1,4 +1,4 @@
-/* $OpenBSD: locore.s,v 1.51 2023/01/31 15:18:51 deraadt Exp $ */
+/* $OpenBSD: locore.s,v 1.52 2023/10/24 13:20:09 claudio Exp $ */
 /* $NetBSD: locore.s,v 1.94 2001/04/26 03:10:44 ross Exp $ */
 
 /*-
@@ -776,9 +776,7 @@ LEAF(cpu_idle_leave, 0)
  * by the s2 register.
  */
 LEAF(proc_trampoline, 0)
-#if defined(MULTIPROCESSOR)
-	CALL(proc_trampoline_mp)
-#endif
+	CALL(proc_trampoline_mi)
 	mov	s0, pv
 	mov	s1, ra
 	mov	s2, a0
diff --git a/sys/arch/alpha/alpha/vm_machdep.c b/sys/arch/alpha/alpha/vm_machdep.c
index 362f9908f9c..b04a12651ea 100644
--- a/sys/arch/alpha/alpha/vm_machdep.c
+++ b/sys/arch/alpha/alpha/vm_machdep.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: vm_machdep.c,v 1.51 2023/04/11 00:45:07 jsg Exp $ */
+/* $OpenBSD: vm_machdep.c,v 1.52 2023/10/24 13:20:09 claudio Exp $ */
 /* $NetBSD: vm_machdep.c,v 1.55 2000/03/29 03:49:48 simonb Exp $ */
 
 /*
@@ -162,15 +162,7 @@ cpu_fork(struct proc *p1, struct proc *p2, void *stack, void *tcb,
 	up->u_pcb.pcb_context[2] = (u_int64_t)arg;
 	up->u_pcb.pcb_context[7] =
 	    (u_int64_t)proc_trampoline;		/* ra: assembly magic */
-#ifdef MULTIPROCESSOR
-	/*
-	 * MULTIPROCESSOR kernels will reuse the IPL of the parent
-	 * process, and will lower to IPL_NONE in proc_trampoline_mp().
-	 */
 	up->u_pcb.pcb_context[8] = IPL_SCHED;	/* ps: IPL */
-#else
-	up->u_pcb.pcb_context[8] = IPL_NONE;	/* ps: IPL */
-#endif
 }
 
 struct kmem_va_mode kv_physwait = {
diff --git a/sys/arch/amd64/amd64/cpu.c b/sys/arch/amd64/amd64/cpu.c
index 60220bff4f5..bee2ff12b6d 100644
--- a/sys/arch/amd64/amd64/cpu.c
+++ b/sys/arch/amd64/amd64/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.175 2023/07/31 04:01:07 guenther Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.176 2023/10/24 13:20:09 claudio Exp $	*/
 /* $NetBSD: cpu.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $ */
 
 /*-
@@ -1074,14 +1074,11 @@ cpu_hatch(void *v)
 	s = splhigh();
 	lcr8(0);
 	intr_enable();
-
-	nanouptime(&ci->ci_schedstate.spc_runtime);
 	splx(s);
 
 	lapic_startclock();
 
-	SCHED_LOCK(s);
-	cpu_switchto(NULL, sched_chooseproc());
+	sched_toidle();
 }
 
 #if defined(DDB)
diff --git a/sys/arch/amd64/amd64/locore.S b/sys/arch/amd64/amd64/locore.S
index 27938da9f28..bcf30f659bd 100644
--- a/sys/arch/amd64/amd64/locore.S
+++ b/sys/arch/amd64/amd64/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.140 2023/07/31 04:01:07 guenther Exp $	*/
+/*	$OpenBSD: locore.S,v 1.141 2023/10/24 13:20:09 claudio Exp $	*/
 /*	$NetBSD: locore.S,v 1.13 2004/03/25 18:33:17 drochner Exp $	*/
 
 /*
@@ -709,10 +709,7 @@ spl_lowered:
 #endif
 
 NENTRY(proc_trampoline)
-#ifdef MULTIPROCESSOR
-	call	proc_trampoline_mp
-#endif
-	movl	$IPL_NONE,CPUVAR(ILEVEL)
+	call	proc_trampoline_mi
 	movq	%r13,%rdi
 	movq	%r12,%rax
 	call	retpoline_rax
diff --git a/sys/arch/arm/arm/cpu.c b/sys/arch/arm/arm/cpu.c
index 0383d68f3d4..cedce406d8d 100644
--- a/sys/arch/arm/arm/cpu.c
+++ b/sys/arch/arm/arm/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.58 2023/06/15 22:18:06 cheloha Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.59 2023/10/24 13:20:09 claudio Exp $	*/
 /*	$NetBSD: cpu.c,v 1.56 2004/04/14 04:01:49 bsh Exp $	*/
 
 
@@ -554,15 +554,12 @@ cpu_start_secondary(struct cpu_info *ci)
 	arm_intr_cpu_enable();
 	cpu_startclock();
 
-	nanouptime(&ci->ci_schedstate.spc_runtime);
-
 	atomic_setbits_int(&ci->ci_flags, CPUF_RUNNING);
 	__asm volatile("dsb sy; sev");
 
 	spllower(IPL_NONE);
 
-	SCHED_LOCK(s);
-	cpu_switchto(NULL, sched_chooseproc());
+	sched_toidle();
 }
 
 void
diff --git a/sys/arch/arm/arm/cpuswitch7.S b/sys/arch/arm/arm/cpuswitch7.S
index a422a6b9486..d7f9e603d93 100644
--- a/sys/arch/arm/arm/cpuswitch7.S
+++ b/sys/arch/arm/arm/cpuswitch7.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpuswitch7.S,v 1.17 2022/12/08 01:25:44 guenther Exp $	*/
+/*	$OpenBSD: cpuswitch7.S,v 1.18 2023/10/24 13:20:09 claudio Exp $	*/
 /*	$NetBSD: cpuswitch.S,v 1.41 2003/11/15 08:44:18 scw Exp $	*/
 
 /*
@@ -221,11 +221,7 @@ ENTRY(savectx)
 
 
 ENTRY(proc_trampoline)
-#ifdef MULTIPROCESSOR
-	bl	proc_trampoline_mp
-#endif
-	mov	r0, #(IPL_NONE)
-	bl	_spllower
+	bl	proc_trampoline_mi
 
 	mov	r0, r5
 	mov	r1, sp
diff --git a/sys/arch/arm64/arm64/cpu.c b/sys/arch/arm64/arm64/cpu.c
index d2d4381d19c..7825d2418fc 100644
--- a/sys/arch/arm64/arm64/cpu.c
+++ b/sys/arch/arm64/arm64/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.98 2023/08/10 19:29:32 kettenis Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.99 2023/10/24 13:20:09 claudio Exp $	*/
 
 /*
  * Copyright (c) 2016 Dale Rahn <drahn@dalerahn.com>
@@ -1176,15 +1176,12 @@ cpu_init_secondary(struct cpu_info *ci)
 	arm_intr_cpu_enable();
 	cpu_startclock();
 
-	nanouptime(&ci->ci_schedstate.spc_runtime);
-
 	atomic_setbits_int(&ci->ci_flags, CPUF_RUNNING);
 	__asm volatile("dsb sy; sev" ::: "memory");
 
 	spllower(IPL_NONE);
 
-	SCHED_LOCK(s);
-	cpu_switchto(NULL, sched_chooseproc());
+	sched_toidle();
 }
 
 void
diff --git a/sys/arch/arm64/arm64/cpuswitch.S b/sys/arch/arm64/arm64/cpuswitch.S
index 5e5b47e40b8..bc95fce90f9 100644
--- a/sys/arch/arm64/arm64/cpuswitch.S
+++ b/sys/arch/arm64/arm64/cpuswitch.S
@@ -1,4 +1,4 @@
-/* $OpenBSD: cpuswitch.S,v 1.6 2022/12/08 01:25:44 guenther Exp $ */
+/* $OpenBSD: cpuswitch.S,v 1.7 2023/10/24 13:20:09 claudio Exp $ */
 /*
  * Copyright (c) 2015 Dale Rahn <drahn@dalerahn.com>
  *
@@ -81,14 +81,8 @@ ENTRY(cpu_switchto_asm)
 	ret
 
 ENTRY(proc_trampoline)
-#ifdef MULTIPROCESSOR
-	bl proc_trampoline_mp
-#endif
-	// call it or just set the variable?
-	mov x0, IPL_NONE
-	bl spllower
+	bl proc_trampoline_mi
 	mov x0, x20
 	blr x19
 	b syscall_return
 
-	
diff --git a/sys/arch/hppa/dev/cpu.c b/sys/arch/hppa/dev/cpu.c
index 777bbd44587..35801a4b8c2 100644
--- a/sys/arch/hppa/dev/cpu.c
+++ b/sys/arch/hppa/dev/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.46 2023/08/29 16:19:34 claudio Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.47 2023/10/24 13:20:09 claudio Exp $	*/
 
 /*
  * Copyright (c) 1998-2003 Michael Shalayeff
@@ -243,7 +243,6 @@ void
 cpu_hatch(void)
 {
 	struct cpu_info *ci = curcpu();
-	int s;
 
 	/* Initialise IPIs. */
 	hppa_ipi_init(ci);
@@ -263,12 +262,7 @@ cpu_hatch(void)
 	while (!start_secondary_cpu)
 		;
 
-	s = splhigh();
-	nanouptime(&ci->ci_schedstate.spc_runtime);
-	splx(s);
-
-	SCHED_LOCK(s);
-	cpu_switchto(NULL, sched_chooseproc());
+	sched_toidle();
 }
 
 void
diff --git a/sys/arch/hppa/hppa/locore.S b/sys/arch/hppa/hppa/locore.S
index a9963b380ee..068440760e1 100644
--- a/sys/arch/hppa/hppa/locore.S
+++ b/sys/arch/hppa/hppa/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.204 2022/10/25 15:15:38 guenther Exp $	*/
+/*	$OpenBSD: locore.S,v 1.205 2023/10/24 13:20:10 claudio Exp $	*/
 
 /*
  * Copyright (c) 1998-2004 Michael Shalayeff
@@ -2697,16 +2697,12 @@ EXIT(cpu_idle_leave)
 
 ENTRY(proc_trampoline,0)
 	copy	r0, r3
-#ifdef MULTIPROCESSOR
 	copy	t4, r5
 	copy	arg0, r4
-	bl	proc_trampoline_mp, rp
+	bl	proc_trampoline_mi, rp
 	nop
 	copy	r4, arg0
 	copy	r5, t4
-#endif
-	mfctl	cr29, t1
-	stw	r0, CI_CPL(t1)
 	.call
 	blr	r0, rp
 	bv,n	r0(t4)
diff --git a/sys/arch/i386/i386/cpu.c b/sys/arch/i386/i386/cpu.c
index 92a293a97a9..3b01d17cd2d 100644
--- a/sys/arch/i386/i386/cpu.c
+++ b/sys/arch/i386/i386/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.113 2023/07/21 04:04:52 guenther Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.114 2023/10/24 13:20:10 claudio Exp $	*/
 /* $NetBSD: cpu.c,v 1.1.2.7 2000/06/26 02:04:05 sommerfeld Exp $ */
 
 /*-
@@ -721,13 +721,11 @@ cpu_hatch(void *v)
 	if (mp_verbose)
 		printf("%s: CPU at apid %ld running\n",
 		    ci->ci_dev->dv_xname, ci->ci_cpuid);
-	nanouptime(&ci->ci_schedstate.spc_runtime);
 	splx(s);
 
 	lapic_startclock();
 
-	SCHED_LOCK(s);
-	cpu_switchto(NULL, sched_chooseproc());
+	sched_toidle();
 }
 
 void
diff --git a/sys/arch/i386/i386/locore.s b/sys/arch/i386/i386/locore.s
index a3511cf4199..2d80d10a7fb 100644
--- a/sys/arch/i386/i386/locore.s
+++ b/sys/arch/i386/i386/locore.s
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.s,v 1.201 2023/04/16 05:40:25 guenther Exp $	*/
+/*	$OpenBSD: locore.s,v 1.202 2023/10/24 13:20:10 claudio Exp $	*/
 /*	$NetBSD: locore.s,v 1.145 1996/05/03 19:41:19 christos Exp $	*/
 
 /*-
@@ -306,10 +306,7 @@ cpu_meltdown: .long		0	# 1 if this CPU has Meltdown
 	.text
 
 NENTRY(proc_trampoline)
-#ifdef MULTIPROCESSOR
-	call	proc_trampoline_mp
-#endif
-	movl	$IPL_NONE,CPL
+	call	proc_trampoline_mi
 	pushl	%ebx
 	call	*%esi
 	addl	$4,%esp
diff --git a/sys/arch/loongson/loongson/machdep.c b/sys/arch/loongson/loongson/machdep.c
index 153b243ad98..b8de66f4c86 100644
--- a/sys/arch/loongson/loongson/machdep.c
+++ b/sys/arch/loongson/loongson/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.100 2022/08/10 12:20:05 miod Exp $ */
+/*	$OpenBSD: machdep.c,v 1.101 2023/10/24 13:20:10 claudio Exp $ */
 
 /*
  * Copyright (c) 2009, 2010, 2014 Miodrag Vallat.
@@ -1267,8 +1267,6 @@ intr_barrier(void *cookie)
 void
 hw_cpu_hatch(struct cpu_info *ci)
 {
-	int s;
-
 	/*
 	 * Set curcpu address on this processor.
 	 */
@@ -1315,8 +1313,7 @@ hw_cpu_hatch(struct cpu_info *ci)
 	spl0();
 	(void)updateimask(0);
 
-	SCHED_LOCK(s);
-	cpu_switchto(NULL, sched_chooseproc());
+	sched_toidle();
 }
 
 void
diff --git a/sys/arch/luna88k/luna88k/machdep.c b/sys/arch/luna88k/luna88k/machdep.c
index fd4c41212c4..daa92792f38 100644
--- a/sys/arch/luna88k/luna88k/machdep.c
+++ b/sys/arch/luna88k/luna88k/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.143 2023/06/15 22:18:07 cheloha Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.144 2023/10/24 13:20:10 claudio Exp $	*/
 /*
  * Copyright (c) 1998, 1999, 2000, 2001 Steve Murphree, Jr.
  * Copyright (c) 1996 Nivas Madhur
@@ -757,14 +757,12 @@ void
 secondary_main()
 {
 	struct cpu_info *ci = curcpu();
-	int s;
 
 	cpu_configuration_print(0);
 	ncpus++;
 
 	clockqueue_init(&ci->ci_queue);
 	sched_init_cpu(ci);
-	nanouptime(&ci->ci_schedstate.spc_runtime);
 	ci->ci_curproc = NULL;
 	ci->ci_randseed = (arc4random() & 0x7fffffff) + 1;
 
@@ -784,12 +782,11 @@ secondary_main()
 	clockintr_cpu_init(NULL);
 
 	spl0();
-	SCHED_LOCK(s);
 	set_psr(get_psr() & ~PSR_IND);
 
 	SET(ci->ci_flags, CIF_ALIVE);
 
-	cpu_switchto(NULL, sched_chooseproc());
+	sched_toidle();
 }
 
 #endif	/* MULTIPROCESSOR */
diff --git a/sys/arch/m88k/m88k/eh_common.S b/sys/arch/m88k/m88k/eh_common.S
index 5f8bfbd81f7..aba3d62c098 100644
--- a/sys/arch/m88k/m88k/eh_common.S
+++ b/sys/arch/m88k/m88k/eh_common.S
@@ -1,4 +1,4 @@
-/*   $OpenBSD: eh_common.S,v 1.64 2022/12/06 18:50:59 guenther Exp $	*/
+/*   $OpenBSD: eh_common.S,v 1.65 2023/10/24 13:20:10 claudio Exp $	*/
 /*
  * Mach Operating System
  * Copyright (c) 1993-1991 Carnegie Mellon University
@@ -2211,12 +2211,7 @@ ASLOCAL(do_softint)
  */
 
 ENTRY(proc_trampoline)
-#ifdef MULTIPROCESSOR
-	bsr	proc_trampoline_mp
-#endif
-
-	bsr.n	setipl			/* setipl(IPL_NONE) */
-	 or	%r2,  %r0,  %r0
+	bsr	proc_trampoline_mi
 
 	ld	%r1,  %r31, 0		/* load func */
 	ld	%r2,  %r31, 4 		/* load arg */
diff --git a/sys/arch/macppc/macppc/cpu.c b/sys/arch/macppc/macppc/cpu.c
index 8a1ca0684a0..f3dbe078fde 100644
--- a/sys/arch/macppc/macppc/cpu.c
+++ b/sys/arch/macppc/macppc/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.86 2023/06/15 22:18:07 cheloha Exp $ */
+/*	$OpenBSD: cpu.c,v 1.87 2023/10/24 13:20:10 claudio Exp $ */
 
 /*
  * Copyright (c) 1997 Per Fogelstrom
@@ -686,7 +686,7 @@ void
 cpu_hatch(void)
 {
 	volatile struct cpu_hatch_data *h = cpu_hatch_data;
-	int intrstate, s;
+	int intrstate;
 
         /* Initialize timebase. */
 	ppc_mttb(0);
@@ -759,10 +759,6 @@ cpu_hatch(void)
 	curcpu()->ci_ipending = 0;
 	curcpu()->ci_cpl = 0;
 
-	s = splhigh();
-	nanouptime(&curcpu()->ci_schedstate.spc_runtime);
-	splx(s);
-
 	intrstate = ppc_intr_disable();
 	cpu_startclock();
 	ppc_intr_enable(intrstate);
@@ -770,7 +766,6 @@ cpu_hatch(void)
 	/* Enable inter-processor interrupts. */
 	openpic_set_priority(curcpu()->ci_cpuid, 14);
 
-	SCHED_LOCK(s);
-	cpu_switchto(NULL, sched_chooseproc());
+	sched_toidle();
 }
 #endif
diff --git a/sys/arch/macppc/macppc/locore.S b/sys/arch/macppc/macppc/locore.S
index dcecca39bf8..c578d9a4a08 100644
--- a/sys/arch/macppc/macppc/locore.S
+++ b/sys/arch/macppc/macppc/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.61 2022/12/08 01:25:45 guenther Exp $	*/
+/*	$OpenBSD: locore.S,v 1.62 2023/10/24 13:20:10 claudio Exp $	*/
 /*	$NetBSD: locore.S,v 1.2 1996/10/16 19:33:09 ws Exp $	*/
 
 /*
@@ -845,11 +845,7 @@ rfi1:	rfi
 	.globl	proc_trampoline
 	.type	proc_trampoline,@function
 proc_trampoline:
-#ifdef MULTIPROCESSOR
-	bl	proc_trampoline_mp
-#endif
-	li	%r3,0
-	bl	lcsplx
+	bl	proc_trampoline_mi
 	mtlr	%r31
 	mr	%r3,%r30
 	blrl				/* jump indirect to r31 */
diff --git a/sys/arch/macppc/macppc/machdep.c b/sys/arch/macppc/macppc/machdep.c
index 1406c18f36a..783ba6bc9aa 100644
--- a/sys/arch/macppc/macppc/machdep.c
+++ b/sys/arch/macppc/macppc/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.199 2022/10/30 17:43:39 guenther Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.200 2023/10/24 13:20:10 claudio Exp $	*/
 /*	$NetBSD: machdep.c,v 1.4 1996/10/16 19:33:11 ws Exp $	*/
 
 /*
@@ -121,7 +121,6 @@ int lid_action = 1;
 int pwr_action = 1;
 
 void dumpsys(void);
-int lcsplx(int ipl);	/* called from LCore */
 void *ppc_intr_establish(void *lcv, pci_intr_handle_t ih, int type,
     int level, int (*func)(void *), void *arg, const char *name);
 
@@ -719,12 +718,6 @@ dumpsys(void)
 
 }
 
-int
-lcsplx(int ipl)
-{
-	return spllower(ipl);
-}
-
 /*
  * Halt or reboot the machine after syncing/dumping according to howto.
  */
diff --git a/sys/arch/mips64/mips64/clock.c b/sys/arch/mips64/mips64/clock.c
index fc23ffa3d20..a90fc304a67 100644
--- a/sys/arch/mips64/mips64/clock.c
+++ b/sys/arch/mips64/mips64/clock.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: clock.c,v 1.53 2023/09/17 14:50:51 cheloha Exp $ */
+/*	$OpenBSD: clock.c,v 1.54 2023/10/24 13:20:10 claudio Exp $ */
 
 /*
  * Copyright (c) 2001-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -253,10 +253,6 @@ cp0_startclock(struct cpu_info *ci)
 	int s;
 
 	if (!CPU_IS_PRIMARY(ci)) {
-		s = splhigh();
-		nanouptime(&ci->ci_schedstate.spc_runtime);
-		splx(s);
-
 		/* try to avoid getting clock interrupts early */
 		cp0_set_compare(cp0_get_count() - 1);
 
diff --git a/sys/arch/mips64/mips64/context.S b/sys/arch/mips64/mips64/context.S
index c4757ef3d89..bbe64ad9ec2 100644
--- a/sys/arch/mips64/mips64/context.S
+++ b/sys/arch/mips64/mips64/context.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: context.S,v 1.65 2023/08/05 05:46:36 guenther Exp $ */
+/*	$OpenBSD: context.S,v 1.66 2023/10/24 13:20:10 claudio Exp $ */
 
 /*
  * Copyright (c) 2002-2003 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -282,19 +282,8 @@ LEAF(proc_trampoline, 0)
 #ifdef DDB
 	move	zero, ra
 #endif
-#ifdef MULTIPROCESSOR
-	jal	proc_trampoline_mp
+	jal	proc_trampoline_mi
 	 NOP
-#endif
-	/*
-	 * Enable interrupts, since we want kernel threads to
-	 * start at spl0 and with interrupts enabled, and these
-	 * won't ``return to userland''.
-	 */
-#ifndef MULTIPROCESSOR	/* done by proc_trampoline_mp() */
-	jal	splx
-	 xor	a0, a0
-#endif
 	jal	updateimask		# Make sure SR imask is updated
 	 xor	a0, a0			# and interrupts enabled
 
diff --git a/sys/arch/octeon/octeon/machdep.c b/sys/arch/octeon/octeon/machdep.c
index 0adc6f84e47..3c781e5dbd0 100644
--- a/sys/arch/octeon/octeon/machdep.c
+++ b/sys/arch/octeon/octeon/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.136 2023/02/04 19:19:36 cheloha Exp $ */
+/*	$OpenBSD: machdep.c,v 1.137 2023/10/24 13:20:10 claudio Exp $ */
 
 /*
  * Copyright (c) 2009, 2010 Miodrag Vallat.
@@ -1302,8 +1302,6 @@ hw_cpu_boot_secondary(struct cpu_info *ci)
 void
 hw_cpu_hatch(struct cpu_info *ci)
 {
-	int s;
-
 	/*
 	 * Set curcpu address on this processor.
 	 */
@@ -1341,7 +1339,6 @@ hw_cpu_hatch(struct cpu_info *ci)
 	spl0();
 	(void)updateimask(0);
 
-	SCHED_LOCK(s);
-	cpu_switchto(NULL, sched_chooseproc());
+	sched_toidle();
 }
 #endif /* MULTIPROCESSOR */
diff --git a/sys/arch/powerpc64/powerpc64/cpu.c b/sys/arch/powerpc64/powerpc64/cpu.c
index 01014b8231e..3f0bd032eab 100644
--- a/sys/arch/powerpc64/powerpc64/cpu.c
+++ b/sys/arch/powerpc64/powerpc64/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.27 2023/08/19 00:47:51 gkoehler Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.28 2023/10/24 13:20:10 claudio Exp $	*/
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -355,15 +355,12 @@ cpu_start_secondary(void)
 	s = splhigh();
 	cpu_startclock();
 
-	nanouptime(&ci->ci_schedstate.spc_runtime);
-
 	atomic_setbits_int(&ci->ci_flags, CPUF_RUNNING);
 	membar_sync();
 
 	spllower(IPL_NONE);
 
-	SCHED_LOCK(s);
-	cpu_switchto(NULL, sched_chooseproc());
+	sched_toidle();
 }
 
 void
diff --git a/sys/arch/powerpc64/powerpc64/locore.S b/sys/arch/powerpc64/powerpc64/locore.S
index 6c455f714a4..d0a21c1d988 100644
--- a/sys/arch/powerpc64/powerpc64/locore.S
+++ b/sys/arch/powerpc64/powerpc64/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.45 2023/08/21 01:35:43 guenther Exp $	*/
+/*	$OpenBSD: locore.S,v 1.46 2023/10/24 13:20:10 claudio Exp $	*/
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -370,11 +370,7 @@ copystr:
 	.globl proc_trampoline
 	.type proc_trampoline, @function
 proc_trampoline:
-#ifdef MULTIPROCESSOR
-	bl	proc_trampoline_mp
-#endif
-	li	%r3, IPL_NONE
-	bl	splx
+	bl	proc_trampoline_mi
 	mr	%r12, %r31
 	mr	%r3, %r30
 	mtctr	%r12
diff --git a/sys/arch/riscv64/riscv64/cpu.c b/sys/arch/riscv64/riscv64/cpu.c
index 8ab705db302..de8c1422dbc 100644
--- a/sys/arch/riscv64/riscv64/cpu.c
+++ b/sys/arch/riscv64/riscv64/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.15 2023/09/19 19:20:33 kettenis Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.16 2023/10/24 13:20:10 claudio Exp $	*/
 
 /*
  * Copyright (c) 2016 Dale Rahn <drahn@dalerahn.com>
@@ -390,16 +390,13 @@ cpu_start_secondary(void)
 	csr_clear(sstatus, SSTATUS_FS_MASK);
 	csr_set(sie, SIE_SSIE);
 
-	nanouptime(&ci->ci_schedstate.spc_runtime);
-
 	atomic_setbits_int(&ci->ci_flags, CPUF_RUNNING);
 	membar_producer();
 
 	spllower(IPL_NONE);
 	intr_enable();
 
-	SCHED_LOCK(s);
-	cpu_switchto(NULL, sched_chooseproc());
+	sched_toidle();
 }
 
 void
diff --git a/sys/arch/riscv64/riscv64/cpuswitch.S b/sys/arch/riscv64/riscv64/cpuswitch.S
index c6a58b89dad..75626003e3c 100644
--- a/sys/arch/riscv64/riscv64/cpuswitch.S
+++ b/sys/arch/riscv64/riscv64/cpuswitch.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpuswitch.S,v 1.7 2022/12/02 12:27:08 jca Exp $	*/
+/*	$OpenBSD: cpuswitch.S,v 1.8 2023/10/24 13:20:10 claudio Exp $	*/
 
 /*
  * Copyright (c) 2015 Dale Rahn <drahn@dalerahn.com>
@@ -92,13 +92,7 @@ ENTRY(cpu_switchto_asm)
 END(cpu_switch_asm)
 
 ENTRY(proc_trampoline)
-#ifdef MULTIPROCESSOR
-	la	t0, proc_trampoline_mp
-	jalr	t0
-#endif
-	// call it or just set the variable?
-	li	a0, IPL_NONE
-	la	t0, spllower
+	la	t0, proc_trampoline_mi
 	jalr	t0
 	mv	a0, s2
 	jalr	s1
diff --git a/sys/arch/sh/sh/locore_subr.S b/sys/arch/sh/sh/locore_subr.S
index 97ed4c4fbef..342868128b0 100644
--- a/sys/arch/sh/sh/locore_subr.S
+++ b/sys/arch/sh/sh/locore_subr.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore_subr.S,v 1.16 2023/08/14 07:40:08 miod Exp $	*/
+/*	$OpenBSD: locore_subr.S,v 1.17 2023/10/24 13:20:10 claudio Exp $	*/
 /*	$NetBSD: locore_subr.S,v 1.28 2006/01/23 22:52:09 uwe Exp $	*/
 
 /*
@@ -471,10 +471,15 @@ NENTRY(_cpu_spin)
  *	respectively. set by cpu_fork().
  */
 NENTRY(proc_trampoline)
+	mov.l	.L_proc_trampoline_mi, r1
+	jsr	@r1			/* proc_trampoline_mi() */
+	 nop
 	jsr	@r12
 	 mov	r11,	r4
 	__EXCEPTION_RETURN
 	/* NOTREACHED */
+.L_proc_trampoline_mi:
+	.long	proc_trampoline_mi
 	SET_ENTRY_SIZE(proc_trampoline)
 
 
diff --git a/sys/arch/sh/sh/vm_machdep.c b/sys/arch/sh/sh/vm_machdep.c
index 5227ab70ee5..85ad32946df 100644
--- a/sys/arch/sh/sh/vm_machdep.c
+++ b/sys/arch/sh/sh/vm_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vm_machdep.c,v 1.17 2022/05/27 18:55:30 kettenis Exp $	*/
+/*	$OpenBSD: vm_machdep.c,v 1.18 2023/10/24 13:20:10 claudio Exp $	*/
 /*	$NetBSD: vm_machdep.c,v 1.53 2006/08/31 16:49:21 matt Exp $	*/
 
 /*
@@ -227,7 +227,8 @@ cpu_fork(struct proc *p1, struct proc *p2, void *stack, void *tcb,
 	 * Enable interrupt when switch frame is restored, since
 	 * kernel thread begin to run without restoring trapframe.
 	 */
-	sf->sf_sr = PSL_MD;		/* kernel mode, interrupt enable */
+	sf->sf_sr = PSL_MD |		/* kernel mode, interrupt enable */
+	    (IPL_SCHED << 4);
 
 #ifdef SH4
 	if (CPU_IS_SH4) {
diff --git a/sys/arch/sparc64/sparc64/cpu.c b/sys/arch/sparc64/sparc64/cpu.c
index be893f6b191..fb696a1fb0c 100644
--- a/sys/arch/sparc64/sparc64/cpu.c
+++ b/sys/arch/sparc64/sparc64/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.74 2023/06/15 22:18:08 cheloha Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.75 2023/10/24 13:20:10 claudio Exp $	*/
 /*	$NetBSD: cpu.c,v 1.13 2001/05/26 21:27:15 chs Exp $ */
 
 /*
@@ -723,21 +723,15 @@ void
 cpu_hatch(void)
 {
 	struct cpu_info *ci = curcpu();
-	int s;
 
 	cpu_init(ci);
 
 	ci->ci_flags |= CPUF_RUNNING;
 	membar_sync();
 
-	s = splhigh();
-	nanouptime(&ci->ci_schedstate.spc_runtime);
-	splx(s);
-
 	cpu_start_clock();
 
-	SCHED_LOCK(s);
-	cpu_switchto(NULL, sched_chooseproc());
+	sched_toidle();
 }
 #endif
 
diff --git a/sys/arch/sparc64/sparc64/locore.s b/sys/arch/sparc64/sparc64/locore.s
index 88f5fe146f0..17ec463d16e 100644
--- a/sys/arch/sparc64/sparc64/locore.s
+++ b/sys/arch/sparc64/sparc64/locore.s
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.s,v 1.201 2023/04/28 18:27:55 cheloha Exp $	*/
+/*	$OpenBSD: locore.s,v 1.202 2023/10/24 13:20:11 claudio Exp $	*/
 /*	$NetBSD: locore.s,v 1.137 2001/08/13 06:10:10 jdolecek Exp $	*/
 
 /*
@@ -6077,7 +6077,7 @@ END(snapshot)
 
 /*
  * cpu_set_kpc() and cpu_fork() arrange for proc_trampoline() to run
- * after after a process gets chosen in switch(). The stack frame will
+ * after a process gets chosen in mi_switch(). The stack frame will
  * contain a function pointer in %l0, and an argument to pass to it in %l2.
  *
  * If the function *(%l0) returns, we arrange for an immediate return
@@ -6085,13 +6085,10 @@ END(snapshot)
  * and when returning a child to user mode after a fork(2).
  */
 ENTRY(proc_trampoline)
-#ifdef MULTIPROCESSOR
 	save	%sp, -CC64FSZ, %sp
-	call	proc_trampoline_mp
+	call	proc_trampoline_mi
 	 nop
 	restore
-#endif
-	wrpr	%g0, 0, %pil		! Reset interrupt level
 	call	%l0			! re-use current frame
 	 mov	%l1, %o0
 
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index 5c6019409f3..1e5fa351f0b 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_fork.c,v 1.252 2023/09/13 14:25:49 claudio Exp $	*/
+/*	$OpenBSD: kern_fork.c,v 1.253 2023/10/24 13:20:11 claudio Exp $	*/
 /*	$NetBSD: kern_fork.c,v 1.29 1996/02/09 18:59:34 christos Exp $	*/
 
 /*
@@ -50,6 +50,7 @@
 #include <sys/acct.h>
 #include <sys/ktrace.h>
 #include <sys/sched.h>
+#include <sys/smr.h>
 #include <sys/sysctl.h>
 #include <sys/pool.h>
 #include <sys/mman.h>
@@ -664,20 +665,37 @@ freepid(pid_t pid)
 	oldpids[idx++ % nitems(oldpids)] = pid;
 }
 
-#if defined(MULTIPROCESSOR)
-/*
- * XXX This is a slight hack to get newly-formed processes to
- * XXX acquire the kernel lock as soon as they run.
- */
+/* Do machine independent parts of switching to a new process */
 void
-proc_trampoline_mp(void)
+proc_trampoline_mi(void)
 {
+	struct schedstate_percpu *spc = &curcpu()->ci_schedstate;
+	struct proc *p = curproc;
+
 	SCHED_ASSERT_LOCKED();
+
+	clear_resched(curcpu());
+
+#if defined(MULTIPROCESSOR)
 	__mp_unlock(&sched_lock);
+#endif
 	spl0();
+
 	SCHED_ASSERT_UNLOCKED();
 	KERNEL_ASSERT_UNLOCKED();
+	assertwaitok();
+	smr_idle();
+
+	/* Start any optional clock interrupts needed by the thread. */
+	if (ISSET(p->p_p->ps_flags, PS_ITIMER)) {
+		atomic_setbits_int(&spc->spc_schedflags, SPCF_ITIMER);
+		clockintr_advance(spc->spc_itimer, hardclock_period);
+	}
+	if (ISSET(p->p_p->ps_flags, PS_PROFIL)) {
+		atomic_setbits_int(&spc->spc_schedflags, SPCF_PROFCLOCK);
+		clockintr_advance(spc->spc_profclock, profclock_period);
+	}
 
+	nanouptime(&spc->spc_runtime);
 	KERNEL_LOCK();
 }
-#endif
diff --git a/sys/kern/kern_sched.c b/sys/kern/kern_sched.c
index c4a00effa39..b10a64c5e80 100644
--- a/sys/kern/kern_sched.c
+++ b/sys/kern/kern_sched.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sched.c,v 1.92 2023/09/19 11:31:51 claudio Exp $	*/
+/*	$OpenBSD: kern_sched.c,v 1.93 2023/10/24 13:20:11 claudio Exp $	*/
 /*
  * Copyright (c) 2007, 2008 Artur Grabowski <art@openbsd.org>
  *
@@ -220,11 +220,28 @@ sched_idle(void *v)
  */
 void
 sched_exit(struct proc *p)
+{
+	struct schedstate_percpu *spc = &curcpu()->ci_schedstate;
+
+	LIST_INSERT_HEAD(&spc->spc_deadproc, p, p_hash);
+
+	KERNEL_ASSERT_LOCKED();
+	sched_toidle();
+}
+
+void
+sched_toidle(void)
 {
 	struct schedstate_percpu *spc = &curcpu()->ci_schedstate;
 	struct proc *idle;
 	int s;
 
+#ifdef MULTIPROCESSOR
+	/* This process no longer needs to hold the kernel lock. */
+	if (_kernel_lock_held())
+		__mp_release_all(&kernel_lock);
+#endif
+
 	if (ISSET(spc->spc_schedflags, SPCF_ITIMER)) {
 		atomic_clearbits_int(&spc->spc_schedflags, SPCF_ITIMER);
 		clockintr_cancel(spc->spc_itimer);
@@ -234,17 +251,16 @@ sched_exit(struct proc *p)
 		clockintr_cancel(spc->spc_profclock);
 	}
 
-	LIST_INSERT_HEAD(&spc->spc_deadproc, p, p_hash);
-
-#ifdef MULTIPROCESSOR
-	/* This process no longer needs to hold the kernel lock. */
-	KERNEL_ASSERT_LOCKED();
-	__mp_release_all(&kernel_lock);
-#endif
+	atomic_clearbits_int(&spc->spc_schedflags, SPCF_SWITCHCLEAR);
 
 	SCHED_LOCK(s);
+
 	idle = spc->spc_idleproc;
 	idle->p_stat = SRUN;
+
+	uvmexp.swtch++;
+	TRACEPOINT(sched, off__cpu, idle->p_tid + THREAD_PID_OFFSET,
+	    idle->p_p->ps_pid);
 	cpu_switchto(NULL, idle);
 	panic("cpu_switchto returned");
 }
@@ -334,14 +350,16 @@ sched_chooseproc(void)
 			}
 		}
 		p = spc->spc_idleproc;
-		KASSERT(p);
-		KASSERT(p->p_wchan == NULL);
+		if (p == NULL)
+			panic("no idleproc set on CPU%d",
+			    CPU_INFO_UNIT(curcpu()));
 		p->p_stat = SRUN;
+		KASSERT(p->p_wchan == NULL);
 		return (p);
 	}
+again:
 #endif
 
-again:
 	if (spc->spc_whichqs) {
 		queue = ffs(spc->spc_whichqs) - 1;
 		p = TAILQ_FIRST(&spc->spc_qs[queue]);
@@ -351,22 +369,9 @@ again:
 			panic("thread %d not in SRUN: %d", p->p_tid, p->p_stat);
 	} else if ((p = sched_steal_proc(curcpu())) == NULL) {
 		p = spc->spc_idleproc;
-		if (p == NULL) {
-                        int s;
-			/*
-			 * We get here if someone decides to switch during
-			 * boot before forking kthreads, bleh.
-			 * This is kind of like a stupid idle loop.
-			 */
-#ifdef MULTIPROCESSOR
-			__mp_unlock(&sched_lock);
-#endif
-			spl0();
-			delay(10);
-			SCHED_LOCK(s);
-			goto again;
-                }
-		KASSERT(p);
+		if (p == NULL)
+			panic("no idleproc set on CPU%d",
+			    CPU_INFO_UNIT(curcpu()));
 		p->p_stat = SRUN;
 	} 
 
diff --git a/sys/sys/sched.h b/sys/sys/sched.h
index 10d7d40ce49..5a07f971893 100644
--- a/sys/sys/sched.h
+++ b/sys/sys/sched.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sched.h,v 1.66 2023/10/17 00:04:02 cheloha Exp $	*/
+/*	$OpenBSD: sched.h,v 1.67 2023/10/24 13:20:11 claudio Exp $	*/
 /* $NetBSD: sched.h,v 1.2 1999/02/28 18:14:58 ross Exp $ */
 
 /*-
@@ -159,6 +159,7 @@ struct cpu_info;
 void sched_init_cpu(struct cpu_info *);
 void sched_idle(void *);
 void sched_exit(struct proc *);
+void sched_toidle(void);
 void mi_switch(void);
 void cpu_switchto(struct proc *, struct proc *);
 struct proc *sched_chooseproc(void);
-- 
2.20.1