Implement per-CPU caching for the page table page (vp) pool and the PTE

descriptor (pted) pool in the arm64 pmap implementation.  This
significantly reduces the side-effects of lock contention on the kernel
map lock that is (incorrectly) translated into excessive page daemon
wakeups.  This is not a perfect solution but it does lead to significant
speedups on machines with many CPU cores.

This requires adding a new pmap_init_percpu() function that gets called
at the point where kernel is ready to set up the per-CPU pool caches.
Dummy implementations of this function are added for all non-arm64
architectures.  Some other architectures can probably benefit from
providing an actual implementation that sets up per-CPU caches for
pmap pools as well.

ok phessler@, claudio@, miod@, patrick@

diff --git a/sys/arch/alpha/include/pmap.h b/sys/arch/alpha/include/pmap.h
index 5eddbfa..af2437c 100644 (file)
--- a/sys/arch/alpha/include/pmap.h
+++ b/sys/arch/alpha/include/pmap.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: pmap.h,v 1.45 2023/04/13 15:23:21 miod Exp $ */
+/* $OpenBSD: pmap.h,v 1.46 2023/12/11 22:12:52 kettenis Exp $ */
 /* $NetBSD: pmap.h,v 1.37 2000/11/19 03:16:35 thorpej Exp $ */
 
 /*-
@@ -158,6 +158,7 @@ void        pmap_do_tlb_shootdown(struct cpu_info *, struct trapframe *);
 #define pmap_update(pmap)              /* nothing (yet) */
 
 #define pmap_proc_iflush(p, va, len)   /* nothing */
+#define pmap_init_percpu()             do { /* nothing */ } while (0)
 #define pmap_unuse_final(p)            /* nothing */
 #define        pmap_remove_holes(vm)           do { /* nothing */ } while (0)
 

diff --git a/sys/arch/amd64/include/pmap.h b/sys/arch/amd64/include/pmap.h
index 7228d09..18b8779 100644 (file)
--- a/sys/arch/amd64/include/pmap.h
+++ b/sys/arch/amd64/include/pmap.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: pmap.h,v 1.86 2023/04/13 15:23:21 miod Exp $  */
+/*     $OpenBSD: pmap.h,v 1.87 2023/12/11 22:12:52 kettenis Exp $      */
 /*     $NetBSD: pmap.h,v 1.1 2003/04/26 18:39:46 fvdl Exp $    */
 
 /*
@@ -375,6 +375,7 @@ extern const long nbpd[], nkptpmax[];
 #define pmap_valid_entry(E)            ((E) & PG_V) /* is PDE or PTE valid? */
 
 #define pmap_proc_iflush(p,va,len)     /* nothing */
+#define pmap_init_percpu()             do { /* nothing */ } while (0)
 #define pmap_unuse_final(p)            /* nothing */
 #define        pmap_remove_holes(vm)           do { /* nothing */ } while (0)
 

diff --git a/sys/arch/arm/include/pmap.h b/sys/arch/arm/include/pmap.h
index 0622bc9..fa3cb73 100644 (file)
--- a/sys/arch/arm/include/pmap.h
+++ b/sys/arch/arm/include/pmap.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: pmap.h,v 1.54 2023/04/13 15:23:22 miod Exp $  */
+/*     $OpenBSD: pmap.h,v 1.55 2023/12/11 22:12:53 kettenis Exp $      */
 /*     $NetBSD: pmap.h,v 1.76 2003/09/06 09:10:46 rearnsha Exp $       */
 
 /*
@@ -241,6 +241,7 @@ extern struct pmap  kernel_pmap_store;
 
 #define        pmap_deactivate(p)              do { /* nothing */ } while (0)
 
+#define pmap_init_percpu()             do { /* nothing */ } while (0)
 #define pmap_unuse_final(p)            do { /* nothing */ } while (0)
 #define        pmap_remove_holes(vm)           do { /* nothing */ } while (0)
 

diff --git a/sys/arch/arm64/arm64/pmap.c b/sys/arch/arm64/arm64/pmap.c
index 7bb689d..543c7ff 100644 (file)
--- a/sys/arch/arm64/arm64/pmap.c
+++ b/sys/arch/arm64/arm64/pmap.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: pmap.c,v 1.99 2023/08/10 19:29:32 kettenis Exp $ */
+/* $OpenBSD: pmap.c,v 1.100 2023/12/11 22:12:53 kettenis Exp $ */
 /*
  * Copyright (c) 2008-2009,2014-2016 Dale Rahn <drahn@dalerahn.com>
  *
@@ -1872,6 +1872,13 @@ pmap_postinit(void)
            npages * PAGE_SIZE, VM_MAP_INTRSAFE, FALSE, NULL);
 }
 
+void
+pmap_init_percpu(void)
+{
+       pool_cache_init(&pmap_pted_pool);
+       pool_cache_init(&pmap_vp_pool);
+}
+
 void
 pmap_update(pmap_t pm)
 {

diff --git a/sys/arch/arm64/include/pmap.h b/sys/arch/arm64/include/pmap.h
index 8d5bbf2..129fa40 100644 (file)
--- a/sys/arch/arm64/include/pmap.h
+++ b/sys/arch/arm64/include/pmap.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: pmap.h,v 1.24 2023/06/10 19:30:48 kettenis Exp $ */
+/* $OpenBSD: pmap.h,v 1.25 2023/12/11 22:12:53 kettenis Exp $ */
 /*
  * Copyright (c) 2008,2009,2014 Dale Rahn <drahn@dalerahn.com>
  *
@@ -101,6 +101,9 @@ extern struct pmap kernel_pmap_;
 
 vaddr_t pmap_bootstrap(long kvo, paddr_t lpt1,  long kernelstart,
     long kernelend, long ram_start, long ram_end);
+void pmap_postinit(void);
+void pmap_init_percpu(void);
+
 void pmap_kenter_cache(vaddr_t va, paddr_t pa, vm_prot_t prot, int cacheable);
 void pmap_page_ro(pmap_t pm, vaddr_t va, vm_prot_t prot);
 void pmap_page_rw(pmap_t pm, vaddr_t va);
@@ -118,7 +121,6 @@ struct pv_entry;
 /* investigate */
 #define pmap_unuse_final(p)            do { /* nothing */ } while (0)
 int    pmap_fault_fixup(pmap_t, vaddr_t, vm_prot_t);
-void pmap_postinit(void);
 
 #define __HAVE_PMAP_MPSAFE_ENTER_COW
 

diff --git a/sys/arch/hppa/include/pmap.h b/sys/arch/hppa/include/pmap.h
index e536e40..0c9b01b 100644 (file)
--- a/sys/arch/hppa/include/pmap.h
+++ b/sys/arch/hppa/include/pmap.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: pmap.h,v 1.55 2023/04/13 15:23:22 miod Exp $  */
+/*     $OpenBSD: pmap.h,v 1.56 2023/12/11 22:12:53 kettenis Exp $      */
 
 /*
  * Copyright (c) 2002-2004 Michael Shalayeff
@@ -111,6 +111,7 @@ struct vm_page *pmap_unmap_direct(vaddr_t);
 #define pmap_is_modified(pg)   pmap_testbit(pg, PG_PMAP_MOD)
 #define pmap_is_referenced(pg) pmap_testbit(pg, PG_PMAP_REF)
 
+#define pmap_init_percpu()             do { /* nothing */ } while (0)
 #define pmap_unuse_final(p)            /* nothing */
 #define        pmap_remove_holes(vm)           do { /* nothing */ } while (0)
 

diff --git a/sys/arch/i386/include/pmap.h b/sys/arch/i386/include/pmap.h
index 7c2f8f4..b56a740 100644 (file)
--- a/sys/arch/i386/include/pmap.h
+++ b/sys/arch/i386/include/pmap.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: pmap.h,v 1.93 2023/04/13 15:23:22 miod Exp $  */
+/*     $OpenBSD: pmap.h,v 1.94 2023/12/11 22:12:53 kettenis Exp $      */
 /*     $NetBSD: pmap.h,v 1.44 2000/04/24 17:18:18 thorpej Exp $        */
 
 /*
@@ -210,6 +210,7 @@ extern struct pool pmap_pv_pool;
 #define pmap_valid_entry(E)            ((E) & PG_V) /* is PDE or PTE valid? */
 
 #define pmap_proc_iflush(p,va,len)     /* nothing */
+#define pmap_init_percpu()             do { /* nothing */ } while (0)
 #define pmap_unuse_final(p)            /* nothing */
 #define        pmap_remove_holes(vm)           do { /* nothing */ } while (0)
 

diff --git a/sys/arch/m88k/include/pmap.h b/sys/arch/m88k/include/pmap.h
index 746ed8e..7359453 100644 (file)
--- a/sys/arch/m88k/include/pmap.h
+++ b/sys/arch/m88k/include/pmap.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: pmap.h,v 1.29 2023/04/13 15:23:22 miod Exp $  */
+/*     $OpenBSD: pmap.h,v 1.30 2023/12/11 22:12:53 kettenis Exp $      */
 /*
  * Mach Operating System
  * Copyright (c) 1991 Carnegie Mellon University
@@ -64,6 +64,7 @@ int   pmap_set_modify(pmap_t, vaddr_t);
 void   pmap_unmap_firmware(void);
 boolean_t pmap_unsetbit(struct vm_page *, int);
 
+#define pmap_init_percpu()             do { /* nothing */ } while (0)
 #define pmap_unuse_final(p)            /* nothing */
 #define        pmap_remove_holes(vm)           do { /* nothing */ } while (0)
 

diff --git a/sys/arch/mips64/include/pmap.h b/sys/arch/mips64/include/pmap.h
index c5e2170..de3dd34 100644 (file)
--- a/sys/arch/mips64/include/pmap.h
+++ b/sys/arch/mips64/include/pmap.h
@@ -1,4 +1,4 @@
-/*      $OpenBSD: pmap.h,v 1.53 2023/01/31 15:18:55 deraadt Exp $ */
+/*      $OpenBSD: pmap.h,v 1.54 2023/12/11 22:12:53 kettenis Exp $ */
 
 /*
  * Copyright (c) 1987 Carnegie-Mellon University
@@ -165,6 +165,7 @@ int pmap_copyinsn(pmap_t, vaddr_t, uint32_t *);
 int    pmap_emulate_modify(pmap_t, vaddr_t);
 void   pmap_page_cache(vm_page_t, u_int);
 
+#define pmap_init_percpu()             do { /* nothing */ } while (0)
 #define        pmap_unuse_final(p)             do { /* nothing yet */ } while (0)
 #define        pmap_remove_holes(vm)           do { /* nothing */ } while (0)
 

diff --git a/sys/arch/powerpc/include/pmap.h b/sys/arch/powerpc/include/pmap.h
index d7bcfd4..80323ba 100644 (file)
--- a/sys/arch/powerpc/include/pmap.h
+++ b/sys/arch/powerpc/include/pmap.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: pmap.h,v 1.64 2023/04/13 15:23:22 miod Exp $  */
+/*     $OpenBSD: pmap.h,v 1.65 2023/12/11 22:12:53 kettenis Exp $      */
 /*     $NetBSD: pmap.h,v 1.1 1996/09/30 16:34:29 ws Exp $      */
 
 /*-
@@ -150,6 +150,7 @@ void pmap_real_memory(vaddr_t *start, vsize_t *size);
 int pte_spill_v(struct pmap *pm, u_int32_t va, u_int32_t dsisr, int exec_fault);
 int reserve_dumppages(caddr_t p);
 
+#define pmap_init_percpu()             do { /* nothing */ } while (0)
 #define pmap_unuse_final(p)            /* nothing */
 #define        pmap_remove_holes(vm)           do { /* nothing */ } while (0)
 

diff --git a/sys/arch/powerpc64/include/pmap.h b/sys/arch/powerpc64/include/pmap.h
index c7f4c9e..6489f4a 100644 (file)
--- a/sys/arch/powerpc64/include/pmap.h
+++ b/sys/arch/powerpc64/include/pmap.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: pmap.h,v 1.18 2021/10/12 18:06:15 kettenis Exp $      */
+/*     $OpenBSD: pmap.h,v 1.19 2023/12/11 22:12:53 kettenis Exp $      */
 
 /*
  * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
@@ -64,6 +64,7 @@ extern struct pmap kernel_pmap_store;
 #define pmap_resident_count(pm) ((pm)->pm_stats.resident_count)
 #define pmap_wired_count(pm)   ((pm)->pm_stats.wired_count)
 
+#define pmap_init_percpu()             do { /* nothing */ } while (0)
 #define pmap_unuse_final(p)
 #define pmap_remove_holes(vm)
 #define pmap_update(pm)

diff --git a/sys/arch/riscv64/include/pmap.h b/sys/arch/riscv64/include/pmap.h
index d686dfe..fcfd60d 100644 (file)
--- a/sys/arch/riscv64/include/pmap.h
+++ b/sys/arch/riscv64/include/pmap.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: pmap.h,v 1.8 2023/11/24 07:18:49 miod Exp $   */
+/*     $OpenBSD: pmap.h,v 1.9 2023/12/11 22:12:53 kettenis Exp $       */
 
 /*
  * Copyright (c) 2019-2020 Brian Bamsch <bbamsch@google.com>
@@ -111,6 +111,7 @@ void pmap_physload_avail(void);
 struct pv_entry;
 
 /* investigate */
+#define pmap_init_percpu()             do { /* nothing */ } while (0)
 #define pmap_unuse_final(p)            do { /* nothing */ } while (0)
 int    pmap_fault_fixup(pmap_t, vaddr_t, vm_prot_t);
 void   pmap_postinit(void);

diff --git a/sys/arch/sh/include/pmap.h b/sys/arch/sh/include/pmap.h
index 32e9e2c..1680d4d 100644 (file)
--- a/sys/arch/sh/include/pmap.h
+++ b/sys/arch/sh/include/pmap.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: pmap.h,v 1.20 2023/04/13 15:23:22 miod Exp $  */
+/*     $OpenBSD: pmap.h,v 1.21 2023/12/11 22:12:53 kettenis Exp $      */
 /*     $NetBSD: pmap.h,v 1.28 2006/04/10 23:12:11 uwe Exp $    */
 
 /*-
@@ -60,6 +60,7 @@ typedef struct pmap {
 extern struct pmap __pmap_kernel;
 
 void pmap_bootstrap(void);
+#define pmap_init_percpu()             do { /* nothing */ } while (0)
 #define        pmap_unuse_final(p)             do { /* nothing */ } while (0)
 #define        pmap_remove_holes(vm)           do { /* nothing */ } while (0)
 #define        pmap_kernel()                   (&__pmap_kernel)

diff --git a/sys/arch/sparc64/include/pmap.h b/sys/arch/sparc64/include/pmap.h
index f9ee018..2c37802 100644 (file)
--- a/sys/arch/sparc64/include/pmap.h
+++ b/sys/arch/sparc64/include/pmap.h
@@ -158,6 +158,7 @@ extern struct pmap kernel_pmap_;
 #define        pmap_update(pm)                 /* nothing (yet) */
 
 #define pmap_proc_iflush(p,va,len)     /* nothing */
+#define pmap_init_percpu()             do { /* nothing */ } while (0)
 
 void   pmap_bootstrap(u_long, u_long, u_int, u_int);
 int    pmap_copyinsn(pmap_t, vaddr_t, uint32_t *);

diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index 59b15fe..665bb55 100644 (file)
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: init_main.c,v 1.322 2023/08/29 16:19:34 claudio Exp $ */
+/*     $OpenBSD: init_main.c,v 1.323 2023/12/11 22:12:53 kettenis Exp $        */
 /*     $NetBSD: init_main.c,v 1.84.4.1 1996/06/02 09:08:06 mrg Exp $   */
 
 /*
@@ -399,6 +399,7 @@ main(void *framep)
        /* Enable per-CPU data. */
        mbcpuinit();
        kqueue_init_percpu();
+       pmap_init_percpu();
        uvm_init_percpu();
        evcount_init_percpu();