From 44e0cbf245051670af619ef55662ce4a84441667 Mon Sep 17 00:00:00 2001 From: cheloha Date: Sat, 5 Aug 2023 20:07:55 +0000 Subject: [PATCH] hardclock(9): move setitimer(2) code into itimer_update() - Move the setitimer(2) code responsible for updating the ITIMER_VIRTUAL and ITIMER_PROF timers from hardclock(9) into a new clock interrupt routine, itimer_update(). itimer_update() is periodic and runs at the same frequency as the hardclock. + Revise itimerdecr() to run within itimer_mtx instead of entering and leaving it. - Each schedstate_percpu has its own itimer_update() handle, spc_itimer. A new scheduler flag, SPCF_ITIMER, indicates whether spc_itimer was started during the last mi_switch() and needs to be stopped during the next mi_switch() or sched_exit(). - A new per-process flag, PS_ITIMER, indicates whether ITIMER_VIRTUAL and/or ITIMER_PROF are running. Checking the flag is easier than entering itimer_mtx to check process.ps_timer[]. The flag is set and cleared in a new helper function, process_reset_itimer_flag(). - In setitimer(), call need_resched() when the state of ITIMER_VIRTUAL or ITIMER_PROF is changed to force an mi_switch() and update spc_itimer. claudio@ notes that ITIMER_PROF could be implemented as a high-res timer using the thread's execution time as a guide for when to interrupt the process and assert SIGPROF. This would probably work really well in single-threaded processes. ITIMER_VIRTUAL would be more difficult to make high-res, though, as you need to exclude time spent in the kernel. Tested on powerpc64 by gkoehler@. With input from claudio@. Thread: https://marc.info/?l=openbsd-tech&m=169038818517101&w=2 ok claudio@ --- sys/kern/kern_clock.c | 31 +------------- sys/kern/kern_clockintr.c | 6 ++- sys/kern/kern_sched.c | 15 ++++++- sys/kern/kern_time.c | 86 ++++++++++++++++++++++++++------------- sys/kern/sched_bsd.c | 16 ++++++-- sys/sys/proc.h | 5 ++- sys/sys/sched.h | 4 +- sys/sys/systm.h | 4 +- sys/sys/time.h | 6 ++- 9 files changed, 104 insertions(+), 69 deletions(-) diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c index 89ac8a1077d..bcd3a9f24d6 100644 --- a/sys/kern/kern_clock.c +++ b/sys/kern/kern_clock.c @@ -1,4 +1,4 @@ -/* $OpenBSD: kern_clock.c,v 1.110 2023/08/01 07:57:55 claudio Exp $ */ +/* $OpenBSD: kern_clock.c,v 1.111 2023/08/05 20:07:55 cheloha Exp $ */ /* $NetBSD: kern_clock.c,v 1.34 1996/06/09 04:51:03 briggs Exp $ */ /*- @@ -105,43 +105,14 @@ initclocks(void) inittimecounter(); } -/* - * hardclock does the accounting needed for ITIMER_PROF and ITIMER_VIRTUAL. - * We don't want to send signals with psignal from hardclock because it makes - * MULTIPROCESSOR locking very complicated. Instead, to use an idea from - * FreeBSD, we set a flag on the thread and when it goes to return to - * userspace it signals itself. - */ - /* * The real-time timer, interrupting hz times per second. */ void hardclock(struct clockframe *frame) { - struct proc *p; struct cpu_info *ci = curcpu(); - p = curproc; - if (p && ((p->p_flag & (P_SYSTEM | P_WEXIT)) == 0)) { - struct process *pr = p->p_p; - - /* - * Run current process's virtual and profile time, as needed. - */ - if (CLKF_USERMODE(frame) && - timespecisset(&pr->ps_timer[ITIMER_VIRTUAL].it_value) && - itimerdecr(&pr->ps_timer[ITIMER_VIRTUAL], tick_nsec) == 0) { - atomic_setbits_int(&p->p_flag, P_ALRMPEND); - need_proftick(p); - } - if (timespecisset(&pr->ps_timer[ITIMER_PROF].it_value) && - itimerdecr(&pr->ps_timer[ITIMER_PROF], tick_nsec) == 0) { - atomic_setbits_int(&p->p_flag, P_PROFPEND); - need_proftick(p); - } - } - if (--ci->ci_schedstate.spc_rrticks <= 0) roundrobin(ci); diff --git a/sys/kern/kern_clockintr.c b/sys/kern/kern_clockintr.c index 0853b10d7f3..5e80c2e20b9 100644 --- a/sys/kern/kern_clockintr.c +++ b/sys/kern/kern_clockintr.c @@ -1,4 +1,4 @@ -/* $OpenBSD: kern_clockintr.c,v 1.29 2023/07/27 17:52:53 cheloha Exp $ */ +/* $OpenBSD: kern_clockintr.c,v 1.30 2023/08/05 20:07:55 cheloha Exp $ */ /* * Copyright (c) 2003 Dale Rahn * Copyright (c) 2020 Mark Kettenis @@ -196,6 +196,10 @@ clockintr_cpu_init(const struct intrclock *ic) * XXX Need to find a better place to do this. We can't do it in * sched_init_cpu() because initclocks() runs after it. */ + if (spc->spc_itimer->cl_expiration == 0) { + clockintr_stagger(spc->spc_itimer, hardclock_period, + multiplier, MAXCPUS); + } if (spc->spc_profclock->cl_expiration == 0) { clockintr_stagger(spc->spc_profclock, profclock_period, multiplier, MAXCPUS); diff --git a/sys/kern/kern_sched.c b/sys/kern/kern_sched.c index aad71cd9241..6faf6af1989 100644 --- a/sys/kern/kern_sched.c +++ b/sys/kern/kern_sched.c @@ -1,4 +1,4 @@ -/* $OpenBSD: kern_sched.c,v 1.83 2023/08/05 12:41:04 claudio Exp $ */ +/* $OpenBSD: kern_sched.c,v 1.84 2023/08/05 20:07:55 cheloha Exp $ */ /* * Copyright (c) 2007, 2008 Artur Grabowski * @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -87,6 +88,14 @@ sched_init_cpu(struct cpu_info *ci) spc->spc_idleproc = NULL; + if (spc->spc_itimer == NULL) { + spc->spc_itimer = clockintr_establish(&ci->ci_queue, + itimer_update); + if (spc->spc_itimer == NULL) { + panic("%s: clockintr_establish itimer_update", + __func__); + } + } if (spc->spc_profclock == NULL) { spc->spc_profclock = clockintr_establish(&ci->ci_queue, profclock); @@ -223,6 +232,10 @@ sched_exit(struct proc *p) timespecsub(&ts, &spc->spc_runtime, &ts); timespecadd(&p->p_rtime, &ts, &p->p_rtime); + if (ISSET(spc->spc_schedflags, SPCF_ITIMER)) { + atomic_clearbits_int(&spc->spc_schedflags, SPCF_ITIMER); + clockintr_cancel(spc->spc_itimer); + } if (ISSET(spc->spc_schedflags, SPCF_PROFCLOCK)) { atomic_clearbits_int(&spc->spc_schedflags, SPCF_PROFCLOCK); clockintr_cancel(spc->spc_profclock); diff --git a/sys/kern/kern_time.c b/sys/kern/kern_time.c index 9caf7c1f327..4a21d54eb83 100644 --- a/sys/kern/kern_time.c +++ b/sys/kern/kern_time.c @@ -1,4 +1,4 @@ -/* $OpenBSD: kern_time.c,v 1.163 2023/02/15 10:07:50 claudio Exp $ */ +/* $OpenBSD: kern_time.c,v 1.164 2023/08/05 20:07:55 cheloha Exp $ */ /* $NetBSD: kern_time.c,v 1.20 1996/02/18 11:57:06 fvdl Exp $ */ /* @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,7 @@ #include #include #include +#include #include #include @@ -52,6 +54,7 @@ #include int itimerfix(struct itimerval *); +void process_reset_itimer_flag(struct process *); /* * Time of day and interval timer support. @@ -551,6 +554,10 @@ setitimer(int which, const struct itimerval *itv, struct itimerval *olditv) timeout_del(&pr->ps_realit_to); } *itimer = its; + if (which == ITIMER_VIRTUAL || which == ITIMER_PROF) { + process_reset_itimer_flag(pr); + need_resched(curcpu()); + } } if (which == ITIMER_REAL) @@ -729,49 +736,72 @@ itimerfix(struct itimerval *itv) } /* - * Decrement an interval timer by the given number of nanoseconds. + * Decrement an interval timer by the given duration. * If the timer expires and it is periodic then reload it. When reloading * the timer we subtract any overrun from the next period so that the timer * does not drift. */ int -itimerdecr(struct itimerspec *itp, long nsec) +itimerdecr(struct itimerspec *itp, const struct timespec *decrement) { - struct timespec decrement; - - NSEC_TO_TIMESPEC(nsec, &decrement); - - mtx_enter(&itimer_mtx); - - /* - * Double-check that the timer is enabled. A different thread - * in setitimer(2) may have disabled it while we were entering - * the mutex. - */ - if (!timespecisset(&itp->it_value)) { - mtx_leave(&itimer_mtx); + timespecsub(&itp->it_value, decrement, &itp->it_value); + if (itp->it_value.tv_sec >= 0 && timespecisset(&itp->it_value)) return (1); - } - - /* - * The timer is enabled. Update and reload it as needed. - */ - timespecsub(&itp->it_value, &decrement, &itp->it_value); - if (itp->it_value.tv_sec >= 0 && timespecisset(&itp->it_value)) { - mtx_leave(&itimer_mtx); - return (1); - } if (!timespecisset(&itp->it_interval)) { timespecclear(&itp->it_value); - mtx_leave(&itimer_mtx); return (0); } while (itp->it_value.tv_sec < 0 || !timespecisset(&itp->it_value)) timespecadd(&itp->it_value, &itp->it_interval, &itp->it_value); - mtx_leave(&itimer_mtx); return (0); } +void +itimer_update(struct clockintr *cl, void *cf) +{ + struct timespec elapsed; + uint64_t nsecs; + struct clockframe *frame = cf; + struct proc *p = curproc; + struct process *pr; + + if (p == NULL || ISSET(p->p_flag, P_SYSTEM | P_WEXIT)) + return; + + pr = p->p_p; + if (!ISSET(pr->ps_flags, PS_ITIMER)) + return; + + nsecs = clockintr_advance(cl, hardclock_period) * hardclock_period; + NSEC_TO_TIMESPEC(nsecs, &elapsed); + + mtx_enter(&itimer_mtx); + if (CLKF_USERMODE(frame) && + timespecisset(&pr->ps_timer[ITIMER_VIRTUAL].it_value) && + itimerdecr(&pr->ps_timer[ITIMER_VIRTUAL], &elapsed) == 0) { + process_reset_itimer_flag(pr); + atomic_setbits_int(&p->p_flag, P_ALRMPEND); + need_proftick(p); + } + if (timespecisset(&pr->ps_timer[ITIMER_PROF].it_value) && + itimerdecr(&pr->ps_timer[ITIMER_PROF], &elapsed) == 0) { + process_reset_itimer_flag(pr); + atomic_setbits_int(&p->p_flag, P_PROFPEND); + need_proftick(p); + } + mtx_leave(&itimer_mtx); +} + +void +process_reset_itimer_flag(struct process *ps) +{ + if (timespecisset(&ps->ps_timer[ITIMER_VIRTUAL].it_value) || + timespecisset(&ps->ps_timer[ITIMER_PROF].it_value)) + atomic_setbits_int(&ps->ps_flags, PS_ITIMER); + else + atomic_clearbits_int(&ps->ps_flags, PS_ITIMER); +} + struct mutex ratecheck_mtx = MUTEX_INITIALIZER(IPL_HIGH); /* diff --git a/sys/kern/sched_bsd.c b/sys/kern/sched_bsd.c index 784e7fde933..d7ab0503059 100644 --- a/sys/kern/sched_bsd.c +++ b/sys/kern/sched_bsd.c @@ -1,4 +1,4 @@ -/* $OpenBSD: sched_bsd.c,v 1.78 2023/07/25 18:16:19 cheloha Exp $ */ +/* $OpenBSD: sched_bsd.c,v 1.79 2023/08/05 20:07:55 cheloha Exp $ */ /* $NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $ */ /*- @@ -350,7 +350,11 @@ mi_switch(void) /* add the time counts for this thread to the process's total */ tuagg_unlocked(pr, p); - /* Stop the profclock if it's running. */ + /* Stop any optional clock interrupts. */ + if (ISSET(spc->spc_schedflags, SPCF_ITIMER)) { + atomic_clearbits_int(&spc->spc_schedflags, SPCF_ITIMER); + clockintr_cancel(spc->spc_itimer); + } if (ISSET(spc->spc_schedflags, SPCF_PROFCLOCK)) { atomic_clearbits_int(&spc->spc_schedflags, SPCF_PROFCLOCK); clockintr_cancel(spc->spc_profclock); @@ -400,7 +404,13 @@ mi_switch(void) */ KASSERT(p->p_cpu == curcpu()); - /* Start the profclock if profil(2) is enabled. */ + /* Start any optional clock interrupts needed by the thread. */ + if (ISSET(p->p_p->ps_flags, PS_ITIMER)) { + atomic_setbits_int(&p->p_cpu->ci_schedstate.spc_schedflags, + SPCF_ITIMER); + clockintr_advance(p->p_cpu->ci_schedstate.spc_itimer, + hardclock_period); + } if (ISSET(p->p_p->ps_flags, PS_PROFIL)) { atomic_setbits_int(&p->p_cpu->ci_schedstate.spc_schedflags, SPCF_PROFCLOCK); diff --git a/sys/sys/proc.h b/sys/sys/proc.h index e2013954966..7d2a0da025b 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -1,4 +1,4 @@ -/* $OpenBSD: proc.h,v 1.346 2023/07/14 07:07:08 claudio Exp $ */ +/* $OpenBSD: proc.h,v 1.347 2023/08/05 20:07:56 cheloha Exp $ */ /* $NetBSD: proc.h,v 1.44 1996/04/22 01:23:21 christos Exp $ */ /*- @@ -282,6 +282,7 @@ struct process { #define PS_ORPHAN 0x00800000 /* Process is on an orphan list */ #define PS_CHROOT 0x01000000 /* Process is chrooted */ #define PS_NOBTCFI 0x02000000 /* No Branch Target CFI */ +#define PS_ITIMER 0x04000000 /* Virtual interval timers running */ #define PS_BITS \ ("\20" "\01CONTROLT" "\02EXEC" "\03INEXEC" "\04EXITING" "\05SUGID" \ @@ -289,7 +290,7 @@ struct process { "\013WAITED" "\014COREDUMP" "\015SINGLEEXIT" "\016SINGLEUNWIND" \ "\017NOZOMBIE" "\020STOPPED" "\021SYSTEM" "\022EMBRYO" "\023ZOMBIE" \ "\024NOBROADCASTKILL" "\025PLEDGE" "\026WXNEEDED" "\027EXECPLEDGE" \ - "\030ORPHAN" "\031CHROOT" "\032NOBTCFI") + "\030ORPHAN" "\031CHROOT" "\032NOBTCFI" "\033ITIMER") struct kcov_dev; diff --git a/sys/sys/sched.h b/sys/sys/sched.h index c53b0843120..c63d96df36d 100644 --- a/sys/sys/sched.h +++ b/sys/sys/sched.h @@ -1,4 +1,4 @@ -/* $OpenBSD: sched.h,v 1.59 2023/08/03 16:12:08 claudio Exp $ */ +/* $OpenBSD: sched.h,v 1.60 2023/08/05 20:07:56 cheloha Exp $ */ /* $NetBSD: sched.h,v 1.2 1999/02/28 18:14:58 ross Exp $ */ /*- @@ -107,6 +107,7 @@ struct schedstate_percpu { u_char spc_curpriority; /* usrpri of curproc */ int spc_rrticks; /* ticks until roundrobin() */ + struct clockintr *spc_itimer; /* [o] itimer_update handle */ struct clockintr *spc_profclock; /* [o] profclock handle */ u_int spc_nrun; /* procs on the run queues */ @@ -138,6 +139,7 @@ struct cpustats { #define SPCF_SHOULDHALT 0x0004 /* CPU should be vacated */ #define SPCF_HALTED 0x0008 /* CPU has been halted */ #define SPCF_PROFCLOCK 0x0010 /* profclock() was started */ +#define SPCF_ITIMER 0x0020 /* itimer_update() was started */ #define SCHED_PPQ (128 / SCHED_NQS) /* priorities per queue */ #define NICE_WEIGHT 2 /* priorities per nice level */ diff --git a/sys/sys/systm.h b/sys/sys/systm.h index 8511338ef59..65e724575f6 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -1,4 +1,4 @@ -/* $OpenBSD: systm.h,v 1.163 2023/07/14 07:07:08 claudio Exp $ */ +/* $OpenBSD: systm.h,v 1.164 2023/08/05 20:07:56 cheloha Exp $ */ /* $NetBSD: systm.h,v 1.50 1996/06/09 04:55:09 briggs Exp $ */ /*- @@ -233,6 +233,8 @@ int tvtohz(const struct timeval *); int tstohz(const struct timespec *); void realitexpire(void *); +extern uint32_t hardclock_period; + struct clockframe; void hardclock(struct clockframe *); void statclock(struct clockframe *); diff --git a/sys/sys/time.h b/sys/sys/time.h index 20bfba4cbb8..c7b9abc2c2e 100644 --- a/sys/sys/time.h +++ b/sys/sys/time.h @@ -1,4 +1,4 @@ -/* $OpenBSD: time.h,v 1.63 2022/12/13 17:30:36 cheloha Exp $ */ +/* $OpenBSD: time.h,v 1.64 2023/08/05 20:07:56 cheloha Exp $ */ /* $NetBSD: time.h,v 1.18 1996/04/23 10:29:33 mycroft Exp $ */ /* @@ -330,8 +330,10 @@ uint64_t getnsecuptime(void); struct proc; int clock_gettime(struct proc *, clockid_t, struct timespec *); +struct clockintr; +void itimer_update(struct clockintr *, void *); + void cancel_all_itimers(void); -int itimerdecr(struct itimerspec *, long); int settime(const struct timespec *); int ratecheck(struct timeval *, const struct timeval *); int ppsratecheck(struct timeval *, int *, int); -- 2.20.1