From 551b33bfef6b7c53ceaddc2f1c22f6a9f6d18050 Mon Sep 17 00:00:00 2001 From: kettenis Date: Mon, 11 Dec 2023 22:12:52 +0000 Subject: [PATCH] Implement per-CPU caching for the page table page (vp) pool and the PTE descriptor (pted) pool in the arm64 pmap implementation. This significantly reduces the side-effects of lock contention on the kernel map lock that is (incorrectly) translated into excessive page daemon wakeups. This is not a perfect solution but it does lead to significant speedups on machines with many CPU cores. This requires adding a new pmap_init_percpu() function that gets called at the point where kernel is ready to set up the per-CPU pool caches. Dummy implementations of this function are added for all non-arm64 architectures. Some other architectures can probably benefit from providing an actual implementation that sets up per-CPU caches for pmap pools as well. ok phessler@, claudio@, miod@, patrick@ --- sys/arch/alpha/include/pmap.h | 3 ++- sys/arch/amd64/include/pmap.h | 3 ++- sys/arch/arm/include/pmap.h | 3 ++- sys/arch/arm64/arm64/pmap.c | 9 ++++++++- sys/arch/arm64/include/pmap.h | 6 ++++-- sys/arch/hppa/include/pmap.h | 3 ++- sys/arch/i386/include/pmap.h | 3 ++- sys/arch/m88k/include/pmap.h | 3 ++- sys/arch/mips64/include/pmap.h | 3 ++- sys/arch/powerpc/include/pmap.h | 3 ++- sys/arch/powerpc64/include/pmap.h | 3 ++- sys/arch/riscv64/include/pmap.h | 3 ++- sys/arch/sh/include/pmap.h | 3 ++- sys/arch/sparc64/include/pmap.h | 1 + sys/kern/init_main.c | 3 ++- 15 files changed, 37 insertions(+), 15 deletions(-) diff --git a/sys/arch/alpha/include/pmap.h b/sys/arch/alpha/include/pmap.h index 5eddbfad573..af2437cfd22 100644 --- a/sys/arch/alpha/include/pmap.h +++ b/sys/arch/alpha/include/pmap.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.h,v 1.45 2023/04/13 15:23:21 miod Exp $ */ +/* $OpenBSD: pmap.h,v 1.46 2023/12/11 22:12:52 kettenis Exp $ */ /* $NetBSD: pmap.h,v 1.37 2000/11/19 03:16:35 thorpej Exp $ */ /*- @@ -158,6 +158,7 @@ void pmap_do_tlb_shootdown(struct cpu_info *, struct trapframe *); #define pmap_update(pmap) /* nothing (yet) */ #define pmap_proc_iflush(p, va, len) /* nothing */ +#define pmap_init_percpu() do { /* nothing */ } while (0) #define pmap_unuse_final(p) /* nothing */ #define pmap_remove_holes(vm) do { /* nothing */ } while (0) diff --git a/sys/arch/amd64/include/pmap.h b/sys/arch/amd64/include/pmap.h index 7228d09ec59..18b8779dce1 100644 --- a/sys/arch/amd64/include/pmap.h +++ b/sys/arch/amd64/include/pmap.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.h,v 1.86 2023/04/13 15:23:21 miod Exp $ */ +/* $OpenBSD: pmap.h,v 1.87 2023/12/11 22:12:52 kettenis Exp $ */ /* $NetBSD: pmap.h,v 1.1 2003/04/26 18:39:46 fvdl Exp $ */ /* @@ -375,6 +375,7 @@ extern const long nbpd[], nkptpmax[]; #define pmap_valid_entry(E) ((E) & PG_V) /* is PDE or PTE valid? */ #define pmap_proc_iflush(p,va,len) /* nothing */ +#define pmap_init_percpu() do { /* nothing */ } while (0) #define pmap_unuse_final(p) /* nothing */ #define pmap_remove_holes(vm) do { /* nothing */ } while (0) diff --git a/sys/arch/arm/include/pmap.h b/sys/arch/arm/include/pmap.h index 0622bc9c253..fa3cb731f4b 100644 --- a/sys/arch/arm/include/pmap.h +++ b/sys/arch/arm/include/pmap.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.h,v 1.54 2023/04/13 15:23:22 miod Exp $ */ +/* $OpenBSD: pmap.h,v 1.55 2023/12/11 22:12:53 kettenis Exp $ */ /* $NetBSD: pmap.h,v 1.76 2003/09/06 09:10:46 rearnsha Exp $ */ /* @@ -241,6 +241,7 @@ extern struct pmap kernel_pmap_store; #define pmap_deactivate(p) do { /* nothing */ } while (0) +#define pmap_init_percpu() do { /* nothing */ } while (0) #define pmap_unuse_final(p) do { /* nothing */ } while (0) #define pmap_remove_holes(vm) do { /* nothing */ } while (0) diff --git a/sys/arch/arm64/arm64/pmap.c b/sys/arch/arm64/arm64/pmap.c index 7bb689d5040..543c7ff1a71 100644 --- a/sys/arch/arm64/arm64/pmap.c +++ b/sys/arch/arm64/arm64/pmap.c @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.c,v 1.99 2023/08/10 19:29:32 kettenis Exp $ */ +/* $OpenBSD: pmap.c,v 1.100 2023/12/11 22:12:53 kettenis Exp $ */ /* * Copyright (c) 2008-2009,2014-2016 Dale Rahn * @@ -1872,6 +1872,13 @@ pmap_postinit(void) npages * PAGE_SIZE, VM_MAP_INTRSAFE, FALSE, NULL); } +void +pmap_init_percpu(void) +{ + pool_cache_init(&pmap_pted_pool); + pool_cache_init(&pmap_vp_pool); +} + void pmap_update(pmap_t pm) { diff --git a/sys/arch/arm64/include/pmap.h b/sys/arch/arm64/include/pmap.h index 8d5bbf2eaa9..129fa4079b0 100644 --- a/sys/arch/arm64/include/pmap.h +++ b/sys/arch/arm64/include/pmap.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.h,v 1.24 2023/06/10 19:30:48 kettenis Exp $ */ +/* $OpenBSD: pmap.h,v 1.25 2023/12/11 22:12:53 kettenis Exp $ */ /* * Copyright (c) 2008,2009,2014 Dale Rahn * @@ -101,6 +101,9 @@ extern struct pmap kernel_pmap_; vaddr_t pmap_bootstrap(long kvo, paddr_t lpt1, long kernelstart, long kernelend, long ram_start, long ram_end); +void pmap_postinit(void); +void pmap_init_percpu(void); + void pmap_kenter_cache(vaddr_t va, paddr_t pa, vm_prot_t prot, int cacheable); void pmap_page_ro(pmap_t pm, vaddr_t va, vm_prot_t prot); void pmap_page_rw(pmap_t pm, vaddr_t va); @@ -118,7 +121,6 @@ struct pv_entry; /* investigate */ #define pmap_unuse_final(p) do { /* nothing */ } while (0) int pmap_fault_fixup(pmap_t, vaddr_t, vm_prot_t); -void pmap_postinit(void); #define __HAVE_PMAP_MPSAFE_ENTER_COW diff --git a/sys/arch/hppa/include/pmap.h b/sys/arch/hppa/include/pmap.h index e536e403cd5..0c9b01b5dc3 100644 --- a/sys/arch/hppa/include/pmap.h +++ b/sys/arch/hppa/include/pmap.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.h,v 1.55 2023/04/13 15:23:22 miod Exp $ */ +/* $OpenBSD: pmap.h,v 1.56 2023/12/11 22:12:53 kettenis Exp $ */ /* * Copyright (c) 2002-2004 Michael Shalayeff @@ -111,6 +111,7 @@ struct vm_page *pmap_unmap_direct(vaddr_t); #define pmap_is_modified(pg) pmap_testbit(pg, PG_PMAP_MOD) #define pmap_is_referenced(pg) pmap_testbit(pg, PG_PMAP_REF) +#define pmap_init_percpu() do { /* nothing */ } while (0) #define pmap_unuse_final(p) /* nothing */ #define pmap_remove_holes(vm) do { /* nothing */ } while (0) diff --git a/sys/arch/i386/include/pmap.h b/sys/arch/i386/include/pmap.h index 7c2f8f4c95c..b56a7408840 100644 --- a/sys/arch/i386/include/pmap.h +++ b/sys/arch/i386/include/pmap.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.h,v 1.93 2023/04/13 15:23:22 miod Exp $ */ +/* $OpenBSD: pmap.h,v 1.94 2023/12/11 22:12:53 kettenis Exp $ */ /* $NetBSD: pmap.h,v 1.44 2000/04/24 17:18:18 thorpej Exp $ */ /* @@ -210,6 +210,7 @@ extern struct pool pmap_pv_pool; #define pmap_valid_entry(E) ((E) & PG_V) /* is PDE or PTE valid? */ #define pmap_proc_iflush(p,va,len) /* nothing */ +#define pmap_init_percpu() do { /* nothing */ } while (0) #define pmap_unuse_final(p) /* nothing */ #define pmap_remove_holes(vm) do { /* nothing */ } while (0) diff --git a/sys/arch/m88k/include/pmap.h b/sys/arch/m88k/include/pmap.h index 746ed8ec8dc..735945313f9 100644 --- a/sys/arch/m88k/include/pmap.h +++ b/sys/arch/m88k/include/pmap.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.h,v 1.29 2023/04/13 15:23:22 miod Exp $ */ +/* $OpenBSD: pmap.h,v 1.30 2023/12/11 22:12:53 kettenis Exp $ */ /* * Mach Operating System * Copyright (c) 1991 Carnegie Mellon University @@ -64,6 +64,7 @@ int pmap_set_modify(pmap_t, vaddr_t); void pmap_unmap_firmware(void); boolean_t pmap_unsetbit(struct vm_page *, int); +#define pmap_init_percpu() do { /* nothing */ } while (0) #define pmap_unuse_final(p) /* nothing */ #define pmap_remove_holes(vm) do { /* nothing */ } while (0) diff --git a/sys/arch/mips64/include/pmap.h b/sys/arch/mips64/include/pmap.h index c5e2170af11..de3dd345390 100644 --- a/sys/arch/mips64/include/pmap.h +++ b/sys/arch/mips64/include/pmap.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.h,v 1.53 2023/01/31 15:18:55 deraadt Exp $ */ +/* $OpenBSD: pmap.h,v 1.54 2023/12/11 22:12:53 kettenis Exp $ */ /* * Copyright (c) 1987 Carnegie-Mellon University @@ -165,6 +165,7 @@ int pmap_copyinsn(pmap_t, vaddr_t, uint32_t *); int pmap_emulate_modify(pmap_t, vaddr_t); void pmap_page_cache(vm_page_t, u_int); +#define pmap_init_percpu() do { /* nothing */ } while (0) #define pmap_unuse_final(p) do { /* nothing yet */ } while (0) #define pmap_remove_holes(vm) do { /* nothing */ } while (0) diff --git a/sys/arch/powerpc/include/pmap.h b/sys/arch/powerpc/include/pmap.h index d7bcfd44850..80323ba6ea9 100644 --- a/sys/arch/powerpc/include/pmap.h +++ b/sys/arch/powerpc/include/pmap.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.h,v 1.64 2023/04/13 15:23:22 miod Exp $ */ +/* $OpenBSD: pmap.h,v 1.65 2023/12/11 22:12:53 kettenis Exp $ */ /* $NetBSD: pmap.h,v 1.1 1996/09/30 16:34:29 ws Exp $ */ /*- @@ -150,6 +150,7 @@ void pmap_real_memory(vaddr_t *start, vsize_t *size); int pte_spill_v(struct pmap *pm, u_int32_t va, u_int32_t dsisr, int exec_fault); int reserve_dumppages(caddr_t p); +#define pmap_init_percpu() do { /* nothing */ } while (0) #define pmap_unuse_final(p) /* nothing */ #define pmap_remove_holes(vm) do { /* nothing */ } while (0) diff --git a/sys/arch/powerpc64/include/pmap.h b/sys/arch/powerpc64/include/pmap.h index c7f4c9e257b..6489f4af094 100644 --- a/sys/arch/powerpc64/include/pmap.h +++ b/sys/arch/powerpc64/include/pmap.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.h,v 1.18 2021/10/12 18:06:15 kettenis Exp $ */ +/* $OpenBSD: pmap.h,v 1.19 2023/12/11 22:12:53 kettenis Exp $ */ /* * Copyright (c) 2020 Mark Kettenis @@ -64,6 +64,7 @@ extern struct pmap kernel_pmap_store; #define pmap_resident_count(pm) ((pm)->pm_stats.resident_count) #define pmap_wired_count(pm) ((pm)->pm_stats.wired_count) +#define pmap_init_percpu() do { /* nothing */ } while (0) #define pmap_unuse_final(p) #define pmap_remove_holes(vm) #define pmap_update(pm) diff --git a/sys/arch/riscv64/include/pmap.h b/sys/arch/riscv64/include/pmap.h index d686dfe6859..fcfd60d63be 100644 --- a/sys/arch/riscv64/include/pmap.h +++ b/sys/arch/riscv64/include/pmap.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.h,v 1.8 2023/11/24 07:18:49 miod Exp $ */ +/* $OpenBSD: pmap.h,v 1.9 2023/12/11 22:12:53 kettenis Exp $ */ /* * Copyright (c) 2019-2020 Brian Bamsch @@ -111,6 +111,7 @@ void pmap_physload_avail(void); struct pv_entry; /* investigate */ +#define pmap_init_percpu() do { /* nothing */ } while (0) #define pmap_unuse_final(p) do { /* nothing */ } while (0) int pmap_fault_fixup(pmap_t, vaddr_t, vm_prot_t); void pmap_postinit(void); diff --git a/sys/arch/sh/include/pmap.h b/sys/arch/sh/include/pmap.h index 32e9e2c9a60..1680d4d1ab2 100644 --- a/sys/arch/sh/include/pmap.h +++ b/sys/arch/sh/include/pmap.h @@ -1,4 +1,4 @@ -/* $OpenBSD: pmap.h,v 1.20 2023/04/13 15:23:22 miod Exp $ */ +/* $OpenBSD: pmap.h,v 1.21 2023/12/11 22:12:53 kettenis Exp $ */ /* $NetBSD: pmap.h,v 1.28 2006/04/10 23:12:11 uwe Exp $ */ /*- @@ -60,6 +60,7 @@ typedef struct pmap { extern struct pmap __pmap_kernel; void pmap_bootstrap(void); +#define pmap_init_percpu() do { /* nothing */ } while (0) #define pmap_unuse_final(p) do { /* nothing */ } while (0) #define pmap_remove_holes(vm) do { /* nothing */ } while (0) #define pmap_kernel() (&__pmap_kernel) diff --git a/sys/arch/sparc64/include/pmap.h b/sys/arch/sparc64/include/pmap.h index f9ee018f8a9..2c37802bcdc 100644 --- a/sys/arch/sparc64/include/pmap.h +++ b/sys/arch/sparc64/include/pmap.h @@ -158,6 +158,7 @@ extern struct pmap kernel_pmap_; #define pmap_update(pm) /* nothing (yet) */ #define pmap_proc_iflush(p,va,len) /* nothing */ +#define pmap_init_percpu() do { /* nothing */ } while (0) void pmap_bootstrap(u_long, u_long, u_int, u_int); int pmap_copyinsn(pmap_t, vaddr_t, uint32_t *); diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index 59b15fe04c1..665bb55471f 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -1,4 +1,4 @@ -/* $OpenBSD: init_main.c,v 1.322 2023/08/29 16:19:34 claudio Exp $ */ +/* $OpenBSD: init_main.c,v 1.323 2023/12/11 22:12:53 kettenis Exp $ */ /* $NetBSD: init_main.c,v 1.84.4.1 1996/06/02 09:08:06 mrg Exp $ */ /* @@ -399,6 +399,7 @@ main(void *framep) /* Enable per-CPU data. */ mbcpuinit(); kqueue_init_percpu(); + pmap_init_percpu(); uvm_init_percpu(); evcount_init_percpu(); -- 2.20.1