Add per-CPU caches to the pmemrange allocator.

The caches are used primarily to reduce contention on uvm_lock_fpageq() during
concurrent page faults.  For the moment only uvm_pagealloc() tries to get a
page from the current CPU's cache.  So on some architectures the caches are
also used by the pmap layer.

Each cache is composed of two magazines, design is borrowed from jeff bonwick
vmem's paper and the implementation is similar to the one of pool_cache from
dlg@.  However there is no depot layer and magazines are refilled directly by
the pmemrange allocator.

Tested by robert@, claudio@ and Laurence Tratt.

ok kettenis@

diff --git a/sys/arch/amd64/include/cpu.h b/sys/arch/amd64/include/cpu.h
index 7bd9a54..a637738 100644 (file)
--- a/sys/arch/amd64/include/cpu.h
+++ b/sys/arch/amd64/include/cpu.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: cpu.h,v 1.165 2024/04/14 09:59:04 kettenis Exp $      */
+/*     $OpenBSD: cpu.h,v 1.166 2024/04/17 13:12:58 mpi Exp $   */
 /*     $NetBSD: cpu.h,v 1.1 2003/04/26 18:39:39 fvdl Exp $     */
 
 /*-
@@ -53,6 +53,7 @@
 #include <sys/sched.h>
 #include <sys/sensors.h>
 #include <sys/srp.h>
+#include <uvm/uvm_percpu.h>
 
 #ifdef _KERNEL
 
@@ -210,6 +211,8 @@ struct cpu_info {
 
 #ifdef MULTIPROCESSOR
        struct srp_hazard       ci_srp_hazards[SRP_HAZARD_NUM];
+#define __HAVE_UVM_PERCPU
+       struct uvm_pmr_cache    ci_uvm;         /* [o] page cache */
 #endif
 
        struct ksensordev       ci_sensordev;

diff --git a/sys/arch/arm64/include/cpu.h b/sys/arch/arm64/include/cpu.h
index d0521a3..ee820bb 100644 (file)
--- a/sys/arch/arm64/include/cpu.h
+++ b/sys/arch/arm64/include/cpu.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: cpu.h,v 1.43 2024/02/25 19:15:50 cheloha Exp $ */
+/* $OpenBSD: cpu.h,v 1.44 2024/04/17 13:12:58 mpi Exp $ */
 /*
  * Copyright (c) 2016 Dale Rahn <drahn@dalerahn.com>
  *
@@ -108,6 +108,7 @@ void        arm32_vector_init(vaddr_t, int);
 #include <sys/device.h>
 #include <sys/sched.h>
 #include <sys/srp.h>
+#include <uvm/uvm_percpu.h>
 
 struct cpu_info {
        struct device           *ci_dev; /* Device corresponding to this CPU */
@@ -161,6 +162,8 @@ struct cpu_info {
 
 #ifdef MULTIPROCESSOR
        struct srp_hazard       ci_srp_hazards[SRP_HAZARD_NUM];
+#define __HAVE_UVM_PERCPU
+       struct uvm_pmr_cache    ci_uvm;
        volatile int            ci_flags;
 
        volatile int            ci_ddb_paused;

diff --git a/sys/arch/i386/include/cpu.h b/sys/arch/i386/include/cpu.h
index ef7db2a..9ad7163 100644 (file)
--- a/sys/arch/i386/include/cpu.h
+++ b/sys/arch/i386/include/cpu.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: cpu.h,v 1.185 2024/02/25 19:15:50 cheloha Exp $       */
+/*     $OpenBSD: cpu.h,v 1.186 2024/04/17 13:12:58 mpi Exp $   */
 /*     $NetBSD: cpu.h,v 1.35 1996/05/05 19:29:26 christos Exp $        */
 
 /*-
@@ -69,6 +69,7 @@
 #include <sys/sched.h>
 #include <sys/sensors.h>
 #include <sys/srp.h>
+#include <uvm/uvm_percpu.h>
 
 struct intrsource;
 
@@ -99,6 +100,8 @@ struct cpu_info {
 
 #if defined(MULTIPROCESSOR)
        struct srp_hazard ci_srp_hazards[SRP_HAZARD_NUM];
+#define __HAVE_UVM_PERCPU
+       struct uvm_pmr_cache    ci_uvm;
 #endif
 
        /*

diff --git a/sys/uvm/uvm_page.c b/sys/uvm/uvm_page.c
index b9e6952..5958c77 100644 (file)
--- a/sys/uvm/uvm_page.c
+++ b/sys/uvm/uvm_page.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: uvm_page.c,v 1.174 2024/02/13 10:16:28 miod Exp $     */
+/*     $OpenBSD: uvm_page.c,v 1.175 2024/04/17 13:12:58 mpi Exp $      */
 /*     $NetBSD: uvm_page.c,v 1.44 2000/11/27 08:40:04 chs Exp $        */
 
 /*
@@ -877,13 +877,11 @@ uvm_pagerealloc_multi(struct uvm_object *obj, voff_t off, vsize_t size,
  * => only one of obj or anon can be non-null
  * => caller must activate/deactivate page if it is not wired.
  */
-
 struct vm_page *
 uvm_pagealloc(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
     int flags)
 {
-       struct vm_page *pg;
-       struct pglist pgl;
+       struct vm_page *pg = NULL;
        int pmr_flags;
 
        KASSERT(obj == NULL || anon == NULL);
@@ -906,13 +904,10 @@ uvm_pagealloc(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
 
        if (flags & UVM_PGA_ZERO)
                pmr_flags |= UVM_PLA_ZERO;
-       TAILQ_INIT(&pgl);
-       if (uvm_pmr_getpages(1, 0, 0, 1, 0, 1, pmr_flags, &pgl) != 0)
-               goto fail;
-
-       pg = TAILQ_FIRST(&pgl);
-       KASSERT(pg != NULL && TAILQ_NEXT(pg, pageq) == NULL);
 
+       pg = uvm_pmr_cache_get(pmr_flags);
+       if (pg == NULL)
+               return NULL;
        uvm_pagealloc_pg(pg, obj, off, anon);
        KASSERT((pg->pg_flags & PG_DEV) == 0);
        if (flags & UVM_PGA_ZERO)
@@ -921,9 +916,6 @@ uvm_pagealloc(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
                atomic_setbits_int(&pg->pg_flags, PG_CLEAN);
 
        return pg;
-
-fail:
-       return NULL;
 }
 
 /*
@@ -1025,7 +1017,7 @@ void
 uvm_pagefree(struct vm_page *pg)
 {
        uvm_pageclean(pg);
-       uvm_pmr_freepages(pg, 1);
+       uvm_pmr_cache_put(pg);
 }
 
 /*

diff --git a/sys/uvm/uvm_pdaemon.c b/sys/uvm/uvm_pdaemon.c
index 58a47c3..a8c4c32 100644 (file)
--- a/sys/uvm/uvm_pdaemon.c
+++ b/sys/uvm/uvm_pdaemon.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: uvm_pdaemon.c,v 1.111 2024/04/10 15:26:18 mpi Exp $   */
+/*     $OpenBSD: uvm_pdaemon.c,v 1.112 2024/04/17 13:12:58 mpi Exp $   */
 /*     $NetBSD: uvm_pdaemon.c,v 1.23 2000/08/20 10:24:14 bjh21 Exp $   */
 
 /*
@@ -262,6 +262,8 @@ uvm_pageout(void *arg)
 #if NDRM > 0
                drmbackoff(size * 2);
 #endif
+               uvm_pmr_cache_drain();
+
                /*
                 * scan if needed
                 */

diff --git a/sys/uvm/uvm_percpu.h b/sys/uvm/uvm_percpu.h
new file mode 100644 (file)
index 0000000..bbf6897
--- /dev/null
+++ b/sys/uvm/uvm_percpu.h
@@ -0,0 +1,48 @@
+/*     $OpenBSD: uvm_percpu.h,v 1.1 2024/04/17 13:12:58 mpi Exp $      */
+
+/*
+ * Copyright (c) 2024 Martin Pieuchot <mpi@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef _UVM_UVM_PCPU_H_
+#define _UVM_UVM_PCPU_H_
+
+struct vm_page;
+
+/*
+ * The number of pages per magazine should be large enough to get rid of the
+ * contention in the pmemrange allocator during concurrent page faults and
+ * small enough to limit fragmentation.
+ */
+#define UVM_PMR_CACHEMAGSZ 8
+
+/*
+ * Magazine
+ */
+struct uvm_pmr_cache_item {
+       struct vm_page          *upci_pages[UVM_PMR_CACHEMAGSZ];
+       int                      upci_npages;   /* # of pages in magazine */
+};
+
+/*
+ * Per-CPU cache of physical pages.
+ */
+struct uvm_pmr_cache {
+       struct uvm_pmr_cache_item upc_magz[2];  /* magazines */
+       int                       upc_actv;     /* index of active magazine */
+
+};
+
+#endif /* _UVM_UVM_PCPU_H_ */

diff --git a/sys/uvm/uvm_pmemrange.c b/sys/uvm/uvm_pmemrange.c
index 128772d..8cd1696 100644 (file)
--- a/sys/uvm/uvm_pmemrange.c
+++ b/sys/uvm/uvm_pmemrange.c
@@ -1,6 +1,7 @@
-/*     $OpenBSD: uvm_pmemrange.c,v 1.63 2023/04/10 04:21:20 jsg Exp $  */
+/*     $OpenBSD: uvm_pmemrange.c,v 1.64 2024/04/17 13:12:58 mpi Exp $  */
 
 /*
+ * Copyright (c) 2024 Martin Pieuchot <mpi@openbsd.org>
  * Copyright (c) 2009, 2010 Ariane van der Steldt <ariane@stack.nl>
  *
  * Permission to use, copy, modify, and distribute this software for any
@@ -1261,6 +1262,28 @@ out:
        return 0;
 }
 
+/*
+ * Acquire a single page.
+ *
+ * flags:      UVM_PLA_* flags
+ * result:     returned page.
+ */
+struct vm_page *
+uvm_pmr_getone(int flags)
+{
+       struct vm_page *pg;
+       struct pglist pgl;
+
+       TAILQ_INIT(&pgl);
+       if (uvm_pmr_getpages(1, 0, 0, 1, 0, 1, flags, &pgl) != 0)
+               return NULL;
+
+       pg = TAILQ_FIRST(&pgl);
+       KASSERT(pg != NULL && TAILQ_NEXT(pg, pageq) == NULL);
+
+       return pg;
+}
+
 /*
  * Free a number of contig pages (invoked by uvm_page_init).
  */
@@ -2190,3 +2213,133 @@ uvm_pagezero_thread(void *arg)
                yield();
        }
 }
+
+#if defined(MULTIPROCESSOR) && defined(__HAVE_UVM_PERCPU)
+int
+uvm_pmr_cache_alloc(struct uvm_pmr_cache_item *upci)
+{
+       struct vm_page *pg;
+       struct pglist pgl;
+       int flags = UVM_PLA_NOWAIT|UVM_PLA_NOWAKE;
+       int npages = UVM_PMR_CACHEMAGSZ;
+
+       KASSERT(upci->upci_npages == 0);
+
+       TAILQ_INIT(&pgl);
+       if (uvm_pmr_getpages(npages, 0, 0, 1, 0, npages, flags, &pgl))
+               return -1;
+
+       while ((pg = TAILQ_FIRST(&pgl)) != NULL) {
+               TAILQ_REMOVE(&pgl, pg, pageq);
+               upci->upci_pages[upci->upci_npages] = pg;
+               upci->upci_npages++;
+       }
+       atomic_add_int(&uvmexp.percpucaches, npages);
+
+       return 0;
+}
+
+struct vm_page *
+uvm_pmr_cache_get(int flags)
+{
+       struct uvm_pmr_cache *upc = &curcpu()->ci_uvm;
+       struct uvm_pmr_cache_item *upci;
+       struct vm_page *pg;
+
+       upci = &upc->upc_magz[upc->upc_actv];
+       if (upci->upci_npages == 0) {
+               unsigned int prev;
+
+               prev = (upc->upc_actv == 0) ?  1 : 0;
+               upci = &upc->upc_magz[prev];
+               if (upci->upci_npages == 0) {
+                       atomic_inc_int(&uvmexp.pcpmiss);
+                       if (uvm_pmr_cache_alloc(upci))
+                               return uvm_pmr_getone(flags);
+               }
+               /* Swap magazines */
+               upc->upc_actv = prev;
+       } else {
+               atomic_inc_int(&uvmexp.pcphit);
+       }
+
+       atomic_dec_int(&uvmexp.percpucaches);
+       upci->upci_npages--;
+       pg = upci->upci_pages[upci->upci_npages];
+
+       if (flags & UVM_PLA_ZERO)
+               uvm_pagezero(pg);
+
+       return pg;
+}
+
+void
+uvm_pmr_cache_free(struct uvm_pmr_cache_item *upci)
+{
+       struct pglist pgl;
+       unsigned int i;
+
+       TAILQ_INIT(&pgl);
+       for (i = 0; i < upci->upci_npages; i++)
+               TAILQ_INSERT_TAIL(&pgl, upci->upci_pages[i], pageq);
+
+       uvm_pmr_freepageq(&pgl);
+
+       atomic_sub_int(&uvmexp.percpucaches, upci->upci_npages);
+       upci->upci_npages = 0;
+       memset(upci->upci_pages, 0, sizeof(upci->upci_pages));
+}
+
+void
+uvm_pmr_cache_put(struct vm_page *pg)
+{
+       struct uvm_pmr_cache *upc = &curcpu()->ci_uvm;
+       struct uvm_pmr_cache_item *upci;
+
+       upci = &upc->upc_magz[upc->upc_actv];
+       if (upci->upci_npages >= UVM_PMR_CACHEMAGSZ) {
+               unsigned int prev;
+
+               prev = (upc->upc_actv == 0) ?  1 : 0;
+               upci = &upc->upc_magz[prev];
+               if (upci->upci_npages > 0)
+                       uvm_pmr_cache_free(upci);
+
+               /* Swap magazines */
+               upc->upc_actv = prev;
+               KASSERT(upci->upci_npages == 0);
+       }
+
+       upci->upci_pages[upci->upci_npages] = pg;
+       upci->upci_npages++;
+       atomic_inc_int(&uvmexp.percpucaches);
+}
+
+void
+uvm_pmr_cache_drain(void)
+{
+       struct uvm_pmr_cache *upc = &curcpu()->ci_uvm;
+
+       uvm_pmr_cache_free(&upc->upc_magz[0]);
+       uvm_pmr_cache_free(&upc->upc_magz[1]);
+}
+
+#else /* !(MULTIPROCESSOR && __HAVE_UVM_PERCPU) */
+
+struct vm_page *
+uvm_pmr_cache_get(int flags)
+{
+       return uvm_pmr_getone(flags);
+}
+
+void
+uvm_pmr_cache_put(struct vm_page *pg)
+{
+       uvm_pmr_freepages(pg, 1);
+}
+
+void
+uvm_pmr_cache_drain(void)
+{
+}
+#endif

diff --git a/sys/uvm/uvm_pmemrange.h b/sys/uvm/uvm_pmemrange.h
index 923807c..3369085 100644 (file)
--- a/sys/uvm/uvm_pmemrange.h
+++ b/sys/uvm/uvm_pmemrange.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: uvm_pmemrange.h,v 1.14 2016/09/16 02:47:09 dlg Exp $  */
+/*     $OpenBSD: uvm_pmemrange.h,v 1.15 2024/04/17 13:12:58 mpi Exp $  */
 
 /*
  * Copyright (c) 2009 Ariane van der Steldt <ariane@stack.nl>
@@ -147,5 +147,9 @@ void                         uvm_pmr_remove(struct uvm_pmemrange *,
 struct vm_page         *uvm_pmr_extract_range(struct uvm_pmemrange *,
                            struct vm_page *, paddr_t, paddr_t,
                            struct pglist *);
+struct vm_page         *uvm_pmr_cache_get(int);
+void                    uvm_pmr_cache_put(struct vm_page *);
+void                    uvm_pmr_cache_drain(void);
+
 
 #endif /* _UVM_UVM_PMEMRANGE_H_ */

diff --git a/sys/uvm/uvmexp.h b/sys/uvm/uvmexp.h
index 5b8b182..dc4994f 100644 (file)
--- a/sys/uvm/uvmexp.h
+++ b/sys/uvm/uvmexp.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: uvmexp.h,v 1.12 2024/03/24 10:29:35 mpi Exp $ */
+/*     $OpenBSD: uvmexp.h,v 1.13 2024/04/17 13:12:58 mpi Exp $ */
 
 #ifndef        _UVM_UVMEXP_
 #define        _UVM_UVMEXP_
@@ -66,7 +66,7 @@ struct uvmexp {
        int zeropages;          /* [F] number of zero'd pages */
        int reserve_pagedaemon; /* [I] # of pages reserved for pagedaemon */
        int reserve_kernel;     /* [I] # of pages reserved for kernel */
-       int unused01;           /* formerly anonpages */
+       int percpucaches;       /* [a] # of pages in per-CPU caches */
        int vnodepages;         /* XXX # of pages used by vnode page cache */
        int vtextpages;         /* XXX # of pages used by vtext vnodes */
 
@@ -101,8 +101,8 @@ struct uvmexp {
        int syscalls;           /* system calls */
        int pageins;            /* [p] pagein operation count */
                                /* pageouts are in pdpageouts below */
-       int unused07;           /* formerly obsolete_swapins */
-       int unused08;           /* formerly obsolete_swapouts */
+       int pcphit;             /* [a] # of pagealloc from per-CPU cache */
+       int pcpmiss;            /* [a] # of times a per-CPU cache was empty */
        int pgswapin;           /* pages swapped in */
        int pgswapout;          /* pages swapped out */
        int forks;              /* forks */

diff --git a/usr.bin/systat/uvm.c b/usr.bin/systat/uvm.c
index 957b4db..4bb759a 100644 (file)
--- a/usr.bin/systat/uvm.c
+++ b/usr.bin/systat/uvm.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: uvm.c,v 1.6 2022/11/27 23:18:54 kn Exp $      */
+/*     $OpenBSD: uvm.c,v 1.7 2024/04/17 13:12:58 mpi Exp $     */
 /*
  * Copyright (c) 2008 Can Erkin Acar <canacar@openbsd.org>
  * Copyright (c) 2018 Kenneth R Westerback <krw@openbsd.org>
@@ -80,11 +80,10 @@ struct uvmline uvmline[] = {
        { &uvmexp.zeropages, &last_uvmexp.zeropages, "zeropages",
          &uvmexp.pageins, &last_uvmexp.pageins, "pageins",
          &uvmexp.fltrelckok, &last_uvmexp.fltrelckok, "fltrelckok" },
-       { &uvmexp.reserve_pagedaemon, &last_uvmexp.reserve_pagedaemon,
-         "reserve_pagedaemon",
+       { &uvmexp.percpucaches, &last_uvmexp.percpucaches, "percpucaches",
          &uvmexp.pgswapin, &last_uvmexp.pgswapin, "pgswapin",
          &uvmexp.fltanget, &last_uvmexp.fltanget, "fltanget" },
-       { &uvmexp.reserve_kernel, &last_uvmexp.reserve_kernel, "reserve_kernel",
+       { NULL, NULL, NULL,
          &uvmexp.pgswapout, &last_uvmexp.pgswapout, "pgswapout",
          &uvmexp.fltanretry, &last_uvmexp.fltanretry, "fltanretry" },
        { NULL, NULL, NULL,
@@ -143,13 +142,13 @@ struct uvmline uvmline[] = {
          NULL, NULL, NULL },
        { &uvmexp.pagesize, &last_uvmexp.pagesize, "pagesize",
          &uvmexp.pdpending, &last_uvmexp.pdpending, "pdpending",
-         NULL, NULL, NULL },
+         NULL, NULL, "Per-CPU Counters" },
        { &uvmexp.pagemask, &last_uvmexp.pagemask, "pagemask",
          &uvmexp.pddeact, &last_uvmexp.pddeact, "pddeact",
-         NULL, NULL, NULL },
+         &uvmexp.pcphit, &last_uvmexp.pcphit, "pcphit" },
        { &uvmexp.pageshift, &last_uvmexp.pageshift, "pageshift",
          NULL, NULL, NULL,
-         NULL, NULL, NULL }
+         &uvmexp.pcpmiss, &last_uvmexp.pcpmiss, "pcpmiss" }
 };
 
 field_def fields_uvm[] = {

diff --git a/usr.bin/vmstat/vmstat.c b/usr.bin/vmstat/vmstat.c
index 870bc9f..a737d36 100644 (file)
--- a/usr.bin/vmstat/vmstat.c
+++ b/usr.bin/vmstat/vmstat.c
@@ -1,5 +1,5 @@
 /*     $NetBSD: vmstat.c,v 1.29.4.1 1996/06/05 00:21:05 cgd Exp $      */
-/*     $OpenBSD: vmstat.c,v 1.155 2022/12/04 23:50:50 cheloha Exp $    */
+/*     $OpenBSD: vmstat.c,v 1.156 2024/04/17 13:12:58 mpi Exp $        */
 
 /*
  * Copyright (c) 1980, 1986, 1991, 1993
@@ -513,7 +513,12 @@ dosum(void)
                     uvmexp.reserve_pagedaemon);
        (void)printf("%11u pages reserved for kernel\n",
                     uvmexp.reserve_kernel);
+       (void)printf("%11u pages in per-cpu caches\n",
+                    uvmexp.percpucaches);
 
+       /* per-cpu cache */
+       (void)printf("%11u per-cpu cache hits\n", uvmexp.pcphit);
+       (void)printf("%11u per-cpu cache misses\n", uvmexp.pcpmiss);
        /* swap */
        (void)printf("%11u swap pages\n", uvmexp.swpages);
        (void)printf("%11u swap pages in use\n", uvmexp.swpginuse);