Add support for per-cpu event counters, to be used for clock and IPI

counters where the event being counted occurs across all CPUs in the
system.  Counter instances can be made per-cpu by calling evcount_percpu()
after the counter is attached, and this can occur before or after all system
CPUs are attached.  Per-cpu counter instances should be incremented using
evcount_inc().

ok kettenis@ jca@ cheloha@

diff --git a/share/man/man9/evcount.9 b/share/man/man9/evcount.9
index 17ba630..e30fa49 100644 (file)
--- a/share/man/man9/evcount.9
+++ b/share/man/man9/evcount.9
@@ -1,19 +1,25 @@
-.\"    $OpenBSD: evcount.9,v 1.7 2018/04/23 10:31:24 dlg Exp $
+.\"    $OpenBSD: evcount.9,v 1.8 2022/11/10 07:05:41 jmatthew Exp $
 .\" Written by Jared Yanovich
 .\" This file belongs to the public domain, 11/02/2004.
-.Dd $Mdocdate: April 23 2018 $
+.Dd $Mdocdate: November 10 2022 $
 .Dt EVCOUNT 9
 .Os
 .Sh NAME
 .Nm evcount ,
 .Nm evcount_attach ,
-.Nm evcount_detach
+.Nm evcount_detach ,
+.Nm evcount_percpu ,
+.Nm evcount_inc
 .Nd generic interrupt and event counter kernel API
 .Sh SYNOPSIS
 .In sys/evcount.h
 .Ft void
 .Fn evcount_attach "struct evcount *ec" "const char *name" "void *data"
 .Ft void
+.Fn evcount_percpu "struct evcount *ec"
+.Ft void
+.Fn evcount_inc "struct evcount *ec"
+.Ft void
 .Fn evcount_detach "struct evcount *ec"
 .Sh DESCRIPTION
 The
@@ -72,6 +78,7 @@ struct evcount {
        int                     ec_id;          /* counter ID */
        const char              *ec_name;       /* counter name */
        void                    *ec_data;       /* user data */
+       struct cpumem           *ec_cpumem;     /* per-cpu counter */
 
        TAILQ_ENTRY(evcount)    next;
 };
@@ -92,6 +99,22 @@ provides a chunk of data that will be made available through the
 call.
 .Pp
 The
+.Fn evcount_percpu ec
+function configures
+.Fa ec
+with one counter for each CPU in the system.
+It should be used when
+.Fa ec Ns 's
+corresponding interrupt can occur simultaneously on multiple CPUs.
+It must be called immediately after
+.Fn evcount_attach
+is called for a given counter.
+.Pp
+The
+.Fn evcount_inc ec
+function increments the given counter.
+.Pp
+The
 .Fn evcount_detach ec
 function removes the given event counter
 .Fa ec
@@ -193,7 +216,8 @@ will be made aware of interrupt occurrence.
 .Xr queue 3 ,
 .Xr intro 4 ,
 .Xr vmstat 8 ,
-.Xr autoconf 9
+.Xr autoconf 9 ,
+.Xr counters_alloc 9
 .Sh AUTHORS
 .An -nosplit
 The

diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index be9348c..b59f026 100644 (file)
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: init_main.c,v 1.318 2022/10/30 17:43:40 guenther Exp $        */
+/*     $OpenBSD: init_main.c,v 1.319 2022/11/10 07:05:41 jmatthew Exp $        */
 /*     $NetBSD: init_main.c,v 1.84.4.1 1996/06/02 09:08:06 mrg Exp $   */
 
 /*
@@ -71,6 +71,7 @@
 #include <sys/pipe.h>
 #include <sys/witness.h>
 #include <sys/smr.h>
+#include <sys/evcount.h>
 
 #include <sys/syscallargs.h>
 
@@ -397,6 +398,7 @@ main(void *framep)
        mbcpuinit();
        kqueue_init_percpu();
        uvm_init_percpu();
+       evcount_init_percpu();
 
        /* init exec */
        init_exec();

diff --git a/sys/kern/subr_evcount.c b/sys/kern/subr_evcount.c
index 82dd6d4..cc9fc58 100644 (file)
--- a/sys/kern/subr_evcount.c
+++ b/sys/kern/subr_evcount.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: subr_evcount.c,v 1.13 2022/08/14 01:58:28 jsg Exp $ */
+/*     $OpenBSD: subr_evcount.c,v 1.14 2022/11/10 07:05:41 jmatthew Exp $ */
 /*
  * Copyright (c) 2004 Artur Grabowski <art@openbsd.org>
  * Copyright (c) 2004 Aaron Campbell <aaron@openbsd.org>
@@ -29,8 +29,12 @@
 #include <sys/evcount.h>
 #include <sys/systm.h>
 #include <sys/sysctl.h>
+#include <sys/percpu.h>
 
 static TAILQ_HEAD(,evcount) evcount_list = TAILQ_HEAD_INITIALIZER(evcount_list);
+static TAILQ_HEAD(,evcount) evcount_percpu_init_list =
+    TAILQ_HEAD_INITIALIZER(evcount_percpu_init_list);
+static int evcount_percpu_done;
 
 void
 evcount_attach(struct evcount *ec, const char *name, void *data)
@@ -44,10 +48,50 @@ evcount_attach(struct evcount *ec, const char *name, void *data)
        TAILQ_INSERT_TAIL(&evcount_list, ec, next);
 }
 
+void
+evcount_percpu(struct evcount *ec)
+{
+       if (evcount_percpu_done == 0) {
+               TAILQ_REMOVE(&evcount_list, ec, next);
+               TAILQ_INSERT_TAIL(&evcount_percpu_init_list, ec, next);
+       } else {
+               ec->ec_percpu = counters_alloc(1);
+               TAILQ_INSERT_TAIL(&evcount_list, ec, next);
+       }
+}
+
+void
+evcount_init_percpu(void)
+{
+       struct evcount *ec;
+
+       KASSERT(evcount_percpu_done == 0);
+       TAILQ_FOREACH(ec, &evcount_percpu_init_list, next) {
+               ec->ec_percpu = counters_alloc(1);
+               counters_add(ec->ec_percpu, 0, ec->ec_count);
+               ec->ec_count = 0;
+       }
+       TAILQ_CONCAT(&evcount_list, &evcount_percpu_init_list, next);
+       evcount_percpu_done = 1;
+}
+
 void
 evcount_detach(struct evcount *ec)
 {
        TAILQ_REMOVE(&evcount_list, ec, next);
+       if (ec->ec_percpu != NULL) {
+               counters_free(ec->ec_percpu, 1);
+               ec->ec_percpu = NULL;
+       }
+}
+
+void
+evcount_inc(struct evcount *ec)
+{
+       if (ec->ec_percpu != NULL)
+               counters_inc(ec->ec_percpu, 0);
+       else
+               ec->ec_count++;
 }
 
 #ifndef        SMALL_KERNEL
@@ -85,9 +129,13 @@ evcount_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp,
        case KERN_INTRCNT_CNT:
                if (ec == NULL)
                        return (ENOENT);
-               s = splhigh();
-               count = ec->ec_count;
-               splx(s);
+               if (ec->ec_percpu != NULL) {
+                       counters_read(ec->ec_percpu, &count, 1);
+               } else {
+                       s = splhigh();
+                       count = ec->ec_count;
+                       splx(s);
+               }
                error = sysctl_rdquad(oldp, oldlenp, NULL, count);
                break;
        case KERN_INTRCNT_NAME:

diff --git a/sys/sys/evcount.h b/sys/sys/evcount.h
index 9124e94..36968fc 100644 (file)
--- a/sys/sys/evcount.h
+++ b/sys/sys/evcount.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: evcount.h,v 1.3 2010/09/20 06:33:46 matthew Exp $ */
+/*     $OpenBSD: evcount.h,v 1.4 2022/11/10 07:05:41 jmatthew Exp $ */
 /*
  * Copyright (c) 2004 Artur Grabowski <art@openbsd.org>
  * Copyright (c) 2004 Aaron Campbell <aaron@openbsd.org>
@@ -32,17 +32,23 @@
 
 #include <sys/queue.h>
 
+struct cpumem;
+
 struct evcount {
        u_int64_t               ec_count;       /* main counter */
        int                     ec_id;          /* counter ID */
        const char              *ec_name;       /* counter name */
        void                    *ec_data;       /* user data */
+       struct cpumem           *ec_percpu;     /* per-cpu counter */
 
        TAILQ_ENTRY(evcount)    next;
 };
 
 void evcount_attach(struct evcount *, const char *, void *);
 void evcount_detach(struct evcount *);
+void evcount_inc(struct evcount *);
+void evcount_init_percpu(void);
+void evcount_percpu(struct evcount *);
 int evcount_sysctl(int *, u_int, void *, size_t *, void *, size_t);
 
 #endif /* _KERNEL */